piecestore

Documentation

Overview

Package piecestore contains the endpoint for responding to requests from the uplinks and satellites. It implements the upload and download protocol, where the counterpart is in uplink. It uses trust packages to establish trusted satellites.

Index

• Variables
• type Config
• type Endpoint
  • func NewEndpoint(log *zap.Logger, ident *identity.FullIdentity, trust *trust.Pool, ...) (*Endpoint, error)
  • func (endpoint *Endpoint) Delete(ctx context.Context, delete *pb.PieceDeleteRequest) (_ *pb.PieceDeleteResponse, err error)
  • func (endpoint *Endpoint) DeletePieces(ctx context.Context, req *pb.DeletePiecesRequest) (_ *pb.DeletePiecesResponse, err error)
  • func (endpoint *Endpoint) Download(stream pb.DRPCPiecestore_DownloadStream) (err error)
  • func (endpoint *Endpoint) Exists(ctx context.Context, req *pb.ExistsRequest) (_ *pb.ExistsResponse, err error)
  • func (endpoint *Endpoint) IsExpired(expiration time.Time) bool
  • func (endpoint *Endpoint) RestoreTrash(ctx context.Context, restoreTrashReq *pb.RestoreTrashRequest) (res *pb.RestoreTrashResponse, err error)
  • func (endpoint *Endpoint) Retain(ctx context.Context, retainReq *pb.RetainRequest) (res *pb.RetainResponse, err error)
  • func (endpoint *Endpoint) RetainBig(stream pb.DRPCPiecestore_RetainBigStream) error
  • func (endpoint *Endpoint) TestLiveRequestCount() int32
  • func (endpoint *Endpoint) Upload(stream pb.DRPCPiecestore_UploadStream) (err error)
  • func (endpoint *Endpoint) VerifyOrder(ctx context.Context, limit *pb.OrderLimit, order *pb.Order, ...) (err error)
  • func (endpoint *Endpoint) VerifyOrderLimitSignature(ctx context.Context, limit *pb.OrderLimit) (err error)
  • func (endpoint *Endpoint) VerifyPieceHash(ctx context.Context, limit *pb.OrderLimit, hash *pb.PieceHash, ...) (err error)
• type OldConfig

Variables

var (
	// ErrVerifyNotAuthorized is returned when the one submitting the action is not authorized to perform that action.
	ErrVerifyNotAuthorized = errs.Class("not authorized")
	// ErrVerifyUntrusted is returned when action is not trusted.
	ErrVerifyUntrusted = errs.Class("untrusted")
	// ErrVerifyDuplicateRequest is returned when serial number has been already used to submit an action.
	ErrVerifyDuplicateRequest = errs.Class("duplicate request")
)

Types

type Config

type Config struct {
	DatabaseDir             string        `help:"directory to store databases. if empty, uses data path" default:""`
	ExpirationGracePeriod   time.Duration `help:"how soon before expiration date should things be considered expired" default:"48h0m0s"`
	MaxConcurrentRequests   int           `help:"how many concurrent requests are allowed, before uploads are rejected. 0 represents unlimited." default:"0"`
	DeleteWorkers           int           `help:"how many piece delete workers" default:"1"`
	DeleteQueueSize         int           `help:"size of the piece delete queue" default:"10000"`
	ExistsCheckWorkers      int           `help:"how many workers to use to check if satellite pieces exists" default:"5"`
	OrderLimitGracePeriod   time.Duration `help:"how long after OrderLimit creation date are OrderLimits no longer accepted" default:"1h0m0s"`
	CacheSyncInterval       time.Duration `help:"how often the space used cache is synced to persistent storage" releaseDefault:"1h0m0s" devDefault:"0h1m0s"`
	PieceScanOnStartup      bool          `help:"if set to true, all pieces disk usage is recalculated on startup" default:"true"`
	StreamOperationTimeout  time.Duration `help:"how long to spend waiting for a stream operation before canceling" default:"30m"`
	RetainTimeBuffer        time.Duration `help:"allows for small differences in the satellite and storagenode clocks" default:"48h0m0s"`
	ReportCapacityThreshold memory.Size   `help:"threshold below which to immediately notify satellite of capacity" default:"500MB" hidden:"true"`
	MaxUsedSerialsSize      memory.Size   `help:"amount of memory allowed for used serials store - once surpassed, serials will be dropped at random" default:"1MB"`

	MinUploadSpeed                    memory.Size   `` /* 189-byte string literal not displayed */
	MinUploadSpeedGraceDuration       time.Duration `` /* 169-byte string literal not displayed */
	MinUploadSpeedCongestionThreshold float64       `` /* 193-byte string literal not displayed */

	Trust trust.Config

	Monitor monitor.Config
	Orders  orders.Config
}

Config defines parameters for piecestore endpoint.

type Endpoint

type Endpoint struct {
	pb.DRPCContactUnimplementedServer
	// contains filtered or unexported fields
}

Endpoint implements uploading, downloading and deleting for a storage node..

architecture: Endpoint

func NewEndpoint

func NewEndpoint(log *zap.Logger, ident *identity.FullIdentity, trust *trust.Pool, monitor *monitor.Service, retain *retain.Service, pingStats pingStatsSource, store *pieces.Store, trashChore *pieces.TrashChore, pieceDeleter *pieces.Deleter, ordersStore *orders.FileStore, usage bandwidth.DB, usedSerials *usedserials.Table, config Config) (*Endpoint, error)

NewEndpoint creates a new piecestore endpoint.

func (*Endpoint) Delete

func (endpoint *Endpoint) Delete(ctx context.Context, delete *pb.PieceDeleteRequest) (_ *pb.PieceDeleteResponse, err error)

Delete handles deleting a piece on piece store requested by uplink.

Deprecated: use DeletePieces instead.

func (*Endpoint) DeletePieces

func (endpoint *Endpoint) DeletePieces(
	ctx context.Context, req *pb.DeletePiecesRequest,
) (_ *pb.DeletePiecesResponse, err error)

DeletePieces delete a list of pieces on satellite request.

func (*Endpoint) Download

func (endpoint *Endpoint) Download(stream pb.DRPCPiecestore_DownloadStream) (err error)

Download handles Downloading a piece on piecestore.

func (*Endpoint) Exists

func (endpoint *Endpoint) Exists(
	ctx context.Context, req *pb.ExistsRequest,
) (_ *pb.ExistsResponse, err error)

Exists check if pieces from the list exists on storage node. Request will accept only connections from trusted satellite and will check pieces only for that satellite.

func (*Endpoint) IsExpired

func (endpoint *Endpoint) IsExpired(expiration time.Time) bool

IsExpired checks whether the date has already expired (with a threshold) at the time of calling this function.

func (*Endpoint) RestoreTrash

func (endpoint *Endpoint) RestoreTrash(ctx context.Context, restoreTrashReq *pb.RestoreTrashRequest) (res *pb.RestoreTrashResponse, err error)

RestoreTrash restores all trashed items for the satellite issuing the call.

func (*Endpoint) Retain

func (endpoint *Endpoint) Retain(ctx context.Context, retainReq *pb.RetainRequest) (res *pb.RetainResponse, err error)

Retain keeps only piece ids specified in the request.

func (*Endpoint) RetainBig

func (endpoint *Endpoint) RetainBig(stream pb.DRPCPiecestore_RetainBigStream) error

RetainBig keeps only piece ids specified in the request, supports big bloom filters.

func (*Endpoint) TestLiveRequestCount

func (endpoint *Endpoint) TestLiveRequestCount() int32

TestLiveRequestCount returns the current number of live requests.

func (*Endpoint) Upload

func (endpoint *Endpoint) Upload(stream pb.DRPCPiecestore_UploadStream) (err error)

Upload handles uploading a piece on piece store.

func (*Endpoint) VerifyOrder

func (endpoint *Endpoint) VerifyOrder(ctx context.Context, limit *pb.OrderLimit, order *pb.Order, largestOrderAmount int64) (err error)

VerifyOrder verifies that the order corresponds to the order limit and has all the necessary fields.

func (*Endpoint) VerifyOrderLimitSignature

func (endpoint *Endpoint) VerifyOrderLimitSignature(ctx context.Context, limit *pb.OrderLimit) (err error)

VerifyOrderLimitSignature verifies that the order limit signature is valid.

func (*Endpoint) VerifyPieceHash

func (endpoint *Endpoint) VerifyPieceHash(ctx context.Context, limit *pb.OrderLimit, hash *pb.PieceHash, expectedHash []byte) (err error)

VerifyPieceHash verifies whether the piece hash is properly signed and matches the locally computed hash.

type OldConfig

type OldConfig struct {
	Path                   string         `help:"path to store data in" default:"$CONFDIR/storage"`
	WhitelistedSatellites  storj.NodeURLs `help:"a comma-separated list of approved satellite node urls (unused)" devDefault:"" releaseDefault:""`
	AllocatedDiskSpace     memory.Size    `user:"true" help:"total allocated disk space in bytes" default:"1TB"`
	AllocatedBandwidth     memory.Size    `user:"true" help:"total allocated bandwidth in bytes (deprecated)" default:"0B"`
	KBucketRefreshInterval time.Duration  `help:"how frequently Kademlia bucket should be refreshed with node stats" default:"1h0m0s"`
}

OldConfig contains everything necessary for a server.