Documentation ¶
Index ¶
- Constants
- Variables
- func EncodeSegmentID(ctx context.Context, segmentID *internalpb.SegmentID) (_ []byte, err error)
- func EncodeStreamID(ctx context.Context, streamID *internalpb.StreamID) (_ []byte, err error)
- func SignSegmentID(ctx context.Context, signer signing.Signer, unsigned *internalpb.SegmentID) (_ *internalpb.SegmentID, err error)
- func SignStreamID(ctx context.Context, signer signing.Signer, unsigned *internalpb.StreamID) (_ *internalpb.StreamID, err error)
- func TrimUserAgent(userAgent []byte) ([]byte, error)
- func VerifySegmentID(ctx context.Context, satellite signing.Signee, signed *internalpb.SegmentID) (err error)
- func VerifyStreamID(ctx context.Context, satellite signing.Signer, signed *internalpb.StreamID) (err error)
- type APIKeys
- type Config
- type Endpoint
- func (endpoint *Endpoint) Batch(ctx context.Context, req *pb.BatchRequest) (resp *pb.BatchResponse, err error)
- func (endpoint *Endpoint) BeginCopyObject(ctx context.Context, req *pb.ObjectBeginCopyRequest) (resp *pb.ObjectBeginCopyResponse, err error)
- func (endpoint *Endpoint) BeginDeleteObject(ctx context.Context, req *pb.ObjectBeginDeleteRequest) (resp *pb.ObjectBeginDeleteResponse, err error)
- func (endpoint *Endpoint) BeginMoveObject(ctx context.Context, req *pb.ObjectBeginMoveRequest) (resp *pb.ObjectBeginMoveResponse, err error)
- func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRequest) (resp *pb.ObjectBeginResponse, err error)
- func (endpoint *Endpoint) BeginSegment(ctx context.Context, req *pb.SegmentBeginRequest) (resp *pb.SegmentBeginResponse, err error)
- func (endpoint *Endpoint) Close() error
- func (endpoint *Endpoint) CommitObject(ctx context.Context, req *pb.ObjectCommitRequest) (resp *pb.ObjectCommitResponse, err error)
- func (endpoint *Endpoint) CommitSegment(ctx context.Context, req *pb.SegmentCommitRequest) (resp *pb.SegmentCommitResponse, err error)
- func (endpoint *Endpoint) CountBuckets(ctx context.Context, projectID uuid.UUID) (count int, err error)
- func (endpoint *Endpoint) CreateBucket(ctx context.Context, req *pb.BucketCreateRequest) (resp *pb.BucketCreateResponse, err error)
- func (endpoint *Endpoint) DeleteBucket(ctx context.Context, req *pb.BucketDeleteRequest) (resp *pb.BucketDeleteResponse, err error)
- func (endpoint *Endpoint) DeleteCommittedObject(ctx context.Context, projectID uuid.UUID, bucket string, ...) (deletedObjects []*pb.Object, err error)
- func (endpoint *Endpoint) DeleteObjectAnyStatus(ctx context.Context, location metabase.ObjectLocation) (deletedObjects []*pb.Object, err error)
- func (endpoint *Endpoint) DeletePart(ctx context.Context, req *pb.PartDeleteRequest) (resp *pb.PartDeleteResponse, err error)
- func (endpoint *Endpoint) DeletePendingObject(ctx context.Context, stream metabase.ObjectStream) (deletedObjects []*pb.Object, err error)
- func (endpoint *Endpoint) DownloadObject(ctx context.Context, req *pb.ObjectDownloadRequest) (resp *pb.ObjectDownloadResponse, err error)
- func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDownloadRequest) (resp *pb.SegmentDownloadResponse, err error)
- func (endpoint *Endpoint) FinishCopyObject(ctx context.Context, req *pb.ObjectFinishCopyRequest) (resp *pb.ObjectFinishCopyResponse, err error)
- func (endpoint *Endpoint) FinishMoveObject(ctx context.Context, req *pb.ObjectFinishMoveRequest) (resp *pb.ObjectFinishMoveResponse, err error)
- func (endpoint *Endpoint) GetBucket(ctx context.Context, req *pb.BucketGetRequest) (resp *pb.BucketGetResponse, err error)
- func (endpoint *Endpoint) GetObject(ctx context.Context, req *pb.ObjectGetRequest) (resp *pb.ObjectGetResponse, err error)
- func (endpoint *Endpoint) GetObjectIPs(ctx context.Context, req *pb.ObjectGetIPsRequest) (resp *pb.ObjectGetIPsResponse, err error)
- func (endpoint *Endpoint) ListBuckets(ctx context.Context, req *pb.BucketListRequest) (resp *pb.BucketListResponse, err error)
- func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListRequest) (resp *pb.ObjectListResponse, err error)
- func (endpoint *Endpoint) ListPendingObjectStreams(ctx context.Context, req *pb.ObjectListPendingStreamsRequest) (resp *pb.ObjectListPendingStreamsResponse, err error)
- func (endpoint *Endpoint) ListSegments(ctx context.Context, req *pb.SegmentListRequest) (resp *pb.SegmentListResponse, err error)
- func (endpoint *Endpoint) MakeInlineSegment(ctx context.Context, req *pb.SegmentMakeInlineRequest) (resp *pb.SegmentMakeInlineResponse, err error)
- func (endpoint *Endpoint) ProjectInfo(ctx context.Context, req *pb.ProjectInfoRequest) (_ *pb.ProjectInfoResponse, err error)
- func (endpoint *Endpoint) RetryBeginSegmentPieces(ctx context.Context, req *pb.RetryBeginSegmentPiecesRequest) (resp *pb.RetryBeginSegmentPiecesResponse, err error)
- func (endpoint *Endpoint) RevokeAPIKey(ctx context.Context, req *pb.RevokeAPIKeyRequest) (resp *pb.RevokeAPIKeyResponse, err error)
- func (endpoint *Endpoint) UpdateObjectMetadata(ctx context.Context, req *pb.ObjectUpdateMetadataRequest) (resp *pb.ObjectUpdateMetadataResponse, err error)
- type ProjectLimitConfig
- type RSConfig
- type RateLimiterConfig
- type UploadLimiterConfig
Constants ¶
const (
// BoltPointerBucket is the string representing the bucket used for `PointerEntries` in BoltDB.
BoltPointerBucket = "pointers"
)
const MaxUserAgentLength = 500
MaxUserAgentLength is the maximum allowable length of the User Agent.
Variables ¶
var ( // Error general metainfo error. Error = errs.Class("metainfo") // ErrNodeAlreadyExists pointer already has a piece for a node err. ErrNodeAlreadyExists = errs.Class("metainfo: node already exists") // ErrBucketNotEmpty is returned when bucket is required to be empty for an operation. ErrBucketNotEmpty = errs.Class("bucket not empty") )
Functions ¶
func EncodeSegmentID ¶ added in v1.17.1
EncodeSegmentID encodes segment ID into bytes for signing.
func EncodeStreamID ¶ added in v1.17.1
EncodeStreamID encodes stream ID into bytes for signing.
func SignSegmentID ¶ added in v1.17.1
func SignSegmentID(ctx context.Context, signer signing.Signer, unsigned *internalpb.SegmentID) (_ *internalpb.SegmentID, err error)
SignSegmentID signs the segment ID using the specified signer. Signer is a satellite.
func SignStreamID ¶ added in v1.17.1
func SignStreamID(ctx context.Context, signer signing.Signer, unsigned *internalpb.StreamID) (_ *internalpb.StreamID, err error)
SignStreamID signs the stream ID using the specified signer. Signer is a satellite.
func TrimUserAgent ¶ added in v1.43.1
TrimUserAgent returns userAgentBytes that consist of only the product portion of the user agent, and is bounded by the maxUserAgentLength.
func VerifySegmentID ¶ added in v1.17.1
func VerifySegmentID(ctx context.Context, satellite signing.Signee, signed *internalpb.SegmentID) (err error)
VerifySegmentID verifies that the signature inside segment ID belongs to the satellite.
func VerifyStreamID ¶ added in v1.17.1
func VerifyStreamID(ctx context.Context, satellite signing.Signer, signed *internalpb.StreamID) (err error)
VerifyStreamID verifies that the signature inside stream ID belongs to the satellite.
Types ¶
type Config ¶ added in v0.11.0
type Config struct { DatabaseURL string `help:"the database connection string to use" default:"postgres://"` MinRemoteSegmentSize memory.Size `default:"1240" testDefault:"0" help:"minimum remote segment size"` // TODO: fix tests to work with 1024 MaxInlineSegmentSize memory.Size `default:"4KiB" help:"maximum inline segment size"` // we have such default value because max value for ObjectKey is 1024(1 Kib) but EncryptedObjectKey // has encryption overhead 16 bytes. So overall size is 1024 + 16 * 16. MaxEncryptedObjectKeyLength int `default:"1280" help:"maximum encrypted object key length"` MaxSegmentSize memory.Size `default:"64MiB" help:"maximum segment size"` MaxMetadataSize memory.Size `default:"2KiB" help:"maximum segment metadata size"` MaxCommitInterval time.Duration `default:"48h" testDefault:"1h" help:"maximum time allowed to pass between creating and committing a segment"` MinPartSize memory.Size `default:"5MiB" testDefault:"0" help:"minimum allowed part size (last part has no minimum size limit)"` MaxNumberOfParts int `default:"10000" help:"maximum number of parts object can contain"` Overlay bool `default:"true" help:"toggle flag if overlay is enabled"` RS RSConfig `` /* 132-byte string literal not displayed */ SegmentLoop segmentloop.Config `help:"segment loop configuration"` RateLimiter RateLimiterConfig `help:"rate limiter configuration"` UploadLimiter UploadLimiterConfig `help:"object upload limiter configuration"` ProjectLimits ProjectLimitConfig `help:"project limit configuration"` PieceDeletion piecedeletion.Config `help:"piece deletion configuration"` // TODO remove this flag when server-side copy implementation will be finished ServerSideCopy bool `help:"enable code for server-side copy, deprecated. please leave this to true." default:"true"` ServerSideCopyDisabled bool `` /* 192-byte string literal not displayed */ // TODO remove when we benchmarking are done and decision is made. TestListingQuery bool `default:"false" help:"test the new query for non-recursive listing"` }
Config is a configuration struct that is everything you need to start a metainfo.
type Endpoint ¶
type Endpoint struct { pb.DRPCMetainfoUnimplementedServer // contains filtered or unexported fields }
Endpoint metainfo endpoint.
architecture: Endpoint
func NewEndpoint ¶
func NewEndpoint(log *zap.Logger, buckets *buckets.Service, metabaseDB *metabase.DB, deletePieces *piecedeletion.Service, orders *orders.Service, cache *overlay.Service, attributions attribution.DB, peerIdentities overlay.PeerIdentities, apiKeys APIKeys, projectUsage *accounting.Service, projectLimits *accounting.ProjectLimitCache, projects console.Projects, satellite signing.Signer, revocations revocation.DB, config Config) (*Endpoint, error)
NewEndpoint creates new metainfo endpoint instance.
func (*Endpoint) Batch ¶ added in v0.17.0
func (endpoint *Endpoint) Batch(ctx context.Context, req *pb.BatchRequest) (resp *pb.BatchResponse, err error)
Batch handle requests sent in batch.
func (*Endpoint) BeginCopyObject ¶ added in v1.50.1
func (endpoint *Endpoint) BeginCopyObject(ctx context.Context, req *pb.ObjectBeginCopyRequest) (resp *pb.ObjectBeginCopyResponse, err error)
BeginCopyObject begins copying object to different key.
func (*Endpoint) BeginDeleteObject ¶ added in v0.16.0
func (endpoint *Endpoint) BeginDeleteObject(ctx context.Context, req *pb.ObjectBeginDeleteRequest) (resp *pb.ObjectBeginDeleteResponse, err error)
BeginDeleteObject begins object deletion process.
func (*Endpoint) BeginMoveObject ¶ added in v1.39.4
func (endpoint *Endpoint) BeginMoveObject(ctx context.Context, req *pb.ObjectBeginMoveRequest) (resp *pb.ObjectBeginMoveResponse, err error)
BeginMoveObject begins moving object to different key.
func (*Endpoint) BeginObject ¶ added in v0.16.0
func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRequest) (resp *pb.ObjectBeginResponse, err error)
BeginObject begins object.
func (*Endpoint) BeginSegment ¶ added in v0.16.0
func (endpoint *Endpoint) BeginSegment(ctx context.Context, req *pb.SegmentBeginRequest) (resp *pb.SegmentBeginResponse, err error)
BeginSegment begins segment uploading.
func (*Endpoint) CommitObject ¶ added in v0.16.0
func (endpoint *Endpoint) CommitObject(ctx context.Context, req *pb.ObjectCommitRequest) (resp *pb.ObjectCommitResponse, err error)
CommitObject commits an object when all its segments have already been committed.
func (*Endpoint) CommitSegment ¶
func (endpoint *Endpoint) CommitSegment(ctx context.Context, req *pb.SegmentCommitRequest) (resp *pb.SegmentCommitResponse, err error)
CommitSegment commits segment after uploading.
func (*Endpoint) CountBuckets ¶ added in v1.9.1
func (endpoint *Endpoint) CountBuckets(ctx context.Context, projectID uuid.UUID) (count int, err error)
CountBuckets returns the number of buckets a project currently has. TODO: add this to the uplink client side.
func (*Endpoint) CreateBucket ¶ added in v0.15.0
func (endpoint *Endpoint) CreateBucket(ctx context.Context, req *pb.BucketCreateRequest) (resp *pb.BucketCreateResponse, err error)
CreateBucket creates a new bucket.
func (*Endpoint) DeleteBucket ¶ added in v0.15.0
func (endpoint *Endpoint) DeleteBucket(ctx context.Context, req *pb.BucketDeleteRequest) (resp *pb.BucketDeleteResponse, err error)
DeleteBucket deletes a bucket.
func (*Endpoint) DeleteCommittedObject ¶ added in v1.26.2
func (endpoint *Endpoint) DeleteCommittedObject( ctx context.Context, projectID uuid.UUID, bucket string, object metabase.ObjectKey, ) (deletedObjects []*pb.Object, err error)
DeleteCommittedObject deletes all the pieces of the storage nodes that belongs to the specified object.
NOTE: this method is exported for being able to individually test it without having import cycles. TODO: see note on DeleteObjectAnyStatus.
func (*Endpoint) DeleteObjectAnyStatus ¶ added in v1.26.2
func (endpoint *Endpoint) DeleteObjectAnyStatus(ctx context.Context, location metabase.ObjectLocation, ) (deletedObjects []*pb.Object, err error)
DeleteObjectAnyStatus deletes all the pieces of the storage nodes that belongs to the specified object.
NOTE: this method is exported for being able to individually test it without having import cycles. TODO regarding the above note: exporting for testing is fine, but we should name it something that will definitely never ever be added to the rpc set in DRPC definitions. If we ever decide to add an RPC method called "DeleteObjectAnyStatus", DRPC interface definitions is all that is standing in the way from someone remotely calling this. We should name this InternalDeleteObjectAnyStatus or something.
func (*Endpoint) DeletePart ¶ added in v1.36.1
func (endpoint *Endpoint) DeletePart(ctx context.Context, req *pb.PartDeleteRequest) (resp *pb.PartDeleteResponse, err error)
DeletePart is a no-op.
It was used to perform the deletion of a single part from satellite db and from storage nodes. We made this method noop because now we can overwrite segments for pending objects. It's returning no error to avoid failures with uplinks that still are using this method.
func (*Endpoint) DeletePendingObject ¶ added in v1.26.2
func (endpoint *Endpoint) DeletePendingObject(ctx context.Context, stream metabase.ObjectStream) (deletedObjects []*pb.Object, err error)
DeletePendingObject deletes all the pieces of the storage nodes that belongs to the specified pending object.
NOTE: this method is exported for being able to individually test it without having import cycles. TODO: see note on DeleteObjectAnyStatus.
func (*Endpoint) DownloadObject ¶ added in v1.27.0
func (endpoint *Endpoint) DownloadObject(ctx context.Context, req *pb.ObjectDownloadRequest) (resp *pb.ObjectDownloadResponse, err error)
DownloadObject gets object information, creates a download for segments and lists the object segments.
func (*Endpoint) DownloadSegment ¶
func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDownloadRequest) (resp *pb.SegmentDownloadResponse, err error)
DownloadSegment returns data necessary to download segment.
func (*Endpoint) FinishCopyObject ¶ added in v1.50.1
func (endpoint *Endpoint) FinishCopyObject(ctx context.Context, req *pb.ObjectFinishCopyRequest) (resp *pb.ObjectFinishCopyResponse, err error)
FinishCopyObject accepts new encryption keys for object copy and updates the corresponding object ObjectKey and segments EncryptedKey.
func (*Endpoint) FinishMoveObject ¶ added in v1.40.3
func (endpoint *Endpoint) FinishMoveObject(ctx context.Context, req *pb.ObjectFinishMoveRequest) (resp *pb.ObjectFinishMoveResponse, err error)
FinishMoveObject accepts new encryption keys for moved object and updates the corresponding object ObjectKey and segments EncryptedKey.
func (*Endpoint) GetBucket ¶ added in v0.15.0
func (endpoint *Endpoint) GetBucket(ctx context.Context, req *pb.BucketGetRequest) (resp *pb.BucketGetResponse, err error)
GetBucket returns a bucket.
func (*Endpoint) GetObject ¶ added in v0.16.0
func (endpoint *Endpoint) GetObject(ctx context.Context, req *pb.ObjectGetRequest) (resp *pb.ObjectGetResponse, err error)
GetObject gets single object metadata.
func (*Endpoint) GetObjectIPs ¶ added in v1.11.1
func (endpoint *Endpoint) GetObjectIPs(ctx context.Context, req *pb.ObjectGetIPsRequest) (resp *pb.ObjectGetIPsResponse, err error)
GetObjectIPs returns the IP addresses of the nodes holding the pieces for the provided object. This is useful for knowing the locations of the pieces.
func (*Endpoint) ListBuckets ¶ added in v0.15.0
func (endpoint *Endpoint) ListBuckets(ctx context.Context, req *pb.BucketListRequest) (resp *pb.BucketListResponse, err error)
ListBuckets returns buckets in a project where the bucket name matches the request cursor.
func (*Endpoint) ListObjects ¶ added in v0.16.0
func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListRequest) (resp *pb.ObjectListResponse, err error)
ListObjects list objects according to specific parameters.
func (*Endpoint) ListPendingObjectStreams ¶ added in v1.22.1
func (endpoint *Endpoint) ListPendingObjectStreams(ctx context.Context, req *pb.ObjectListPendingStreamsRequest) (resp *pb.ObjectListPendingStreamsResponse, err error)
ListPendingObjectStreams list pending objects according to specific parameters.
func (*Endpoint) ListSegments ¶
func (endpoint *Endpoint) ListSegments(ctx context.Context, req *pb.SegmentListRequest) (resp *pb.SegmentListResponse, err error)
ListSegments list object segments.
func (*Endpoint) MakeInlineSegment ¶ added in v0.16.0
func (endpoint *Endpoint) MakeInlineSegment(ctx context.Context, req *pb.SegmentMakeInlineRequest) (resp *pb.SegmentMakeInlineResponse, err error)
MakeInlineSegment makes inline segment on satellite.
func (*Endpoint) ProjectInfo ¶ added in v0.14.0
func (endpoint *Endpoint) ProjectInfo(ctx context.Context, req *pb.ProjectInfoRequest) (_ *pb.ProjectInfoResponse, err error)
ProjectInfo returns allowed ProjectInfo for the provided API key.
func (*Endpoint) RetryBeginSegmentPieces ¶ added in v1.72.2
func (endpoint *Endpoint) RetryBeginSegmentPieces(ctx context.Context, req *pb.RetryBeginSegmentPiecesRequest) (resp *pb.RetryBeginSegmentPiecesResponse, err error)
RetryBeginSegmentPieces replaces put order limits for failed piece uploads.
func (*Endpoint) RevokeAPIKey ¶ added in v1.7.1
func (endpoint *Endpoint) RevokeAPIKey(ctx context.Context, req *pb.RevokeAPIKeyRequest) (resp *pb.RevokeAPIKeyResponse, err error)
RevokeAPIKey handles requests to revoke an api key.
func (*Endpoint) UpdateObjectMetadata ¶ added in v1.34.1
func (endpoint *Endpoint) UpdateObjectMetadata(ctx context.Context, req *pb.ObjectUpdateMetadataRequest) (resp *pb.ObjectUpdateMetadataResponse, err error)
UpdateObjectMetadata replaces object metadata.
type ProjectLimitConfig ¶ added in v1.9.1
type ProjectLimitConfig struct {
MaxBuckets int `help:"max bucket count for a project." default:"100" testDefault:"10"`
}
ProjectLimitConfig is a configuration struct for default project limits.
type RSConfig ¶ added in v0.14.0
RSConfig is a configuration struct that keeps details about default redundancy strategy information.
Can be used as a flag.
func (*RSConfig) RedundancyStrategy ¶ added in v1.53.1
func (rs *RSConfig) RedundancyStrategy() (eestream.RedundancyStrategy, error)
RedundancyStrategy creates eestream.RedundancyStrategy from config values.
func (*RSConfig) Set ¶ added in v1.17.1
Set sets the value from a string in the format k/m/o/n-size (min/repair/optimal/total-erasuresharesize).
type RateLimiterConfig ¶ added in v0.31.0
type RateLimiterConfig struct { Enabled bool `help:"whether rate limiting is enabled." releaseDefault:"true" devDefault:"true"` Rate float64 `help:"request rate per project per second." releaseDefault:"100" devDefault:"100" testDefault:"1000"` CacheCapacity int `help:"number of projects to cache." releaseDefault:"10000" devDefault:"10" testDefault:"100"` CacheExpiration time.Duration `help:"how long to cache the projects limiter." releaseDefault:"10m" devDefault:"10s"` }
RateLimiterConfig is a configuration struct for endpoint rate limiting.
type UploadLimiterConfig ¶ added in v1.77.2
type UploadLimiterConfig struct { Enabled bool `help:"whether rate limiting is enabled." releaseDefault:"true" devDefault:"true"` SingleObjectLimit time.Duration `help:"how often we can upload to the single object (the same location) per API instance" default:"1s" devDefault:"1ms"` CacheCapacity int `help:"number of object locations to cache." releaseDefault:"10000" devDefault:"10" testDefault:"100"` }
UploadLimiterConfig is a configuration struct for endpoint upload limiting.
Source Files ¶
Directories ¶
Path | Synopsis |
---|---|
Package expireddeletion contains the functions needed to run expired segment deletion
|
Package expireddeletion contains the functions needed to run expired segment deletion |
Package piecedeletion implements service for deleting pieces that combines concurrent requests.
|
Package piecedeletion implements service for deleting pieces that combines concurrent requests. |
Package pointerverification implements verification of pointers.
|
Package pointerverification implements verification of pointers. |