piecestore

Documentation

Index

• Variables
• func GetPieceHashAlgo(ctx context.Context) (algo pb.PieceHashAlgorithm)
• func RetainRequestFromStream(stream pb.DRPCPiecestore_RetainBigStream) (pb.RetainRequest, error)
• func WithPieceHashAlgo(ctx context.Context, hash pb.PieceHashAlgorithm) context.Context
• type Client
  • func Dial(ctx context.Context, dialer rpc.Dialer, nodeURL storj.NodeURL, config Config) (*Client, error)
  • func DialReplaySafe(ctx context.Context, dialer rpc.Dialer, nodeURL storj.NodeURL, config Config) (*Client, error)
  • func (client *Client) Close() error
  • func (client *Client) Download(ctx context.Context, limit *pb.OrderLimit, ...) (_ *Download, err error)
  • func (client *Client) GetPeerIdentity() (*identity.PeerIdentity, error)
  • func (client *Client) NodeURL() storj.NodeURL
  • func (client *Client) Retain(ctx context.Context, req *pb.RetainRequest) (err error)
  • func (client *Client) UploadReader(ctx context.Context, limit *pb.OrderLimit, ...) (hash *pb.PieceHash, err error)
  • func (client *Client) VerifyPieceHash(ctx context.Context, peer *identity.PeerIdentity, limit *pb.OrderLimit, ...) (err error)
• type Config
• type Download
  • func (client *Download) Close() error
  • func (client *Download) GetHashAndLimit() (*pb.PieceHash, *pb.OrderLimit)
  • func (client *Download) Read(data []byte) (read int, err error)
• type ReadBuffer
  • func (buffer *ReadBuffer) Empty() bool
  • func (buffer *ReadBuffer) Error() error
  • func (buffer *ReadBuffer) Errored() bool
  • func (buffer *ReadBuffer) Fill(data []byte)
  • func (buffer *ReadBuffer) IncludeError(err error)
  • func (buffer *ReadBuffer) Read(data []byte) (n int, err error)

Variables

var (
	// Error is the default error class for piecestore client.
	Error = errs.Class("piecestore")

	// CloseError is the error class used for errors generated during a
	// stream close in a piecestore client.
	//
	// Errors of this type should also be wrapped with Error, for backwards
	// compatibility.
	CloseError = errs.Class("piecestore close")
)

var (
	// ErrInternal is an error class for internal errors.
	ErrInternal = errs.Class("internal")
	// ErrProtocol is an error class for unexpected protocol sequence.
	ErrProtocol = errs.Class("protocol")
	// ErrVerifyUntrusted is an error in case there is a trust issue.
	ErrVerifyUntrusted = errs.Class("untrusted")
	// ErrStorageNodeInvalidResponse is an error when a storage node returns a response with invalid data.
	ErrStorageNodeInvalidResponse = errs.Class("storage node has returned an invalid response")
)

var DefaultConfig = Config{
	DownloadBufferSize: 256 * memory.KiB.Int64(),
	UploadBufferSize:   64 * memory.KiB.Int64(),

	InitialStep:      256 * memory.KiB.Int64(),
	MaximumStep:      550 * memory.KiB.Int64(),
	MaximumChunkSize: 16 * memory.KiB.Int32(),

	MessageTimeout: 10 * time.Minute,
}

DefaultConfig are the default params used for upload and download.

var NoiseEnabled = os.Getenv("STORJ_NOISE_DISABLED_EXPERIMENTAL") != "true"

NoiseEnabled indicates whether Noise is enabled in this build.

Functions

func GetPieceHashAlgo

func GetPieceHashAlgo(ctx context.Context) (algo pb.PieceHashAlgorithm)

GetPieceHashAlgo returns with the piece hash algorithm which may be overridden.

func RetainRequestFromStream

func RetainRequestFromStream(stream pb.DRPCPiecestore_RetainBigStream) (pb.RetainRequest, error)

RetainRequestFromStream is the inverse logic of Client.Retain method, which splits the retain messages to smaller chunks. strictly speaking, it's a server side code, but it's easier to maintain and test here, as it should be the opposite of the Retain method.

func WithPieceHashAlgo

func WithPieceHashAlgo(ctx context.Context, hash pb.PieceHashAlgorithm) context.Context

WithPieceHashAlgo sets the used piece hash algorithm.

Types

type Client

type Client struct {
	UploadHashAlgo pb.PieceHashAlgorithm
	// contains filtered or unexported fields
}

Client implements uploading, downloading and deleting content from a piecestore.

func Dial

func Dial(ctx context.Context, dialer rpc.Dialer, nodeURL storj.NodeURL, config Config) (*Client, error)

Dial dials the target piecestore endpoint.

func DialReplaySafe

func DialReplaySafe(ctx context.Context, dialer rpc.Dialer, nodeURL storj.NodeURL, config Config) (*Client, error)

DialReplaySafe dials the target piecestore endpoint for replay safe request types.

func (*Client) Close

func (client *Client) Close() error

Close closes the underlying connection.

func (*Client) Download

func (client *Client) Download(ctx context.Context, limit *pb.OrderLimit, piecePrivateKey storj.PiecePrivateKey, offset, size int64) (_ *Download, err error)

Download starts a new download using the specified order limit at the specified offset and size.

func (*Client) GetPeerIdentity

func (client *Client) GetPeerIdentity() (*identity.PeerIdentity, error)

GetPeerIdentity gets the connection's peer identity. This doesn't work on Noise-based connections.

func (*Client) NodeURL

func (client *Client) NodeURL() storj.NodeURL

NodeURL returns the Node we dialed.

func (*Client) Retain

func (client *Client) Retain(ctx context.Context, req *pb.RetainRequest) (err error)

Retain uses a bloom filter to tell the piece store which pieces to keep.

func (*Client) UploadReader

func (client *Client) UploadReader(ctx context.Context, limit *pb.OrderLimit, piecePrivateKey storj.PiecePrivateKey, data io.Reader) (hash *pb.PieceHash, err error)

UploadReader uploads to the storage node.

func (*Client) VerifyPieceHash

func (client *Client) VerifyPieceHash(ctx context.Context, peer *identity.PeerIdentity, limit *pb.OrderLimit, hash *pb.PieceHash, expectedHash []byte, algorithm pb.PieceHashAlgorithm) (err error)

VerifyPieceHash verifies piece hash which is sent by peer.

type Config

type Config struct {
	DownloadBufferSize int64
	UploadBufferSize   int64

	InitialStep      int64
	MaximumStep      int64
	MaximumChunkSize int32

	MessageTimeout time.Duration
}

Config defines piecestore client parameters for upload and download.

type Download

type Download struct {
	// contains filtered or unexported fields
}

Download implements downloading from a piecestore.

func (*Download) Close

func (client *Download) Close() error

Close closes the downloading.

func (*Download) GetHashAndLimit

func (client *Download) GetHashAndLimit() (*pb.PieceHash, *pb.OrderLimit)

GetHashAndLimit gets the download's hash and original order limit.

func (*Download) Read

func (client *Download) Read(data []byte) (read int, err error)

Read downloads data from the storage node allocating as necessary.

type ReadBuffer

type ReadBuffer struct {
	// contains filtered or unexported fields
}

ReadBuffer implements buffered reading with an error.

func (*ReadBuffer) Empty

func (buffer *ReadBuffer) Empty() bool

Empty checks whether buffer needs to be filled.

func (*ReadBuffer) Error

func (buffer *ReadBuffer) Error() error

Error returns an error if it was encountered.

func (*ReadBuffer) Errored

func (buffer *ReadBuffer) Errored() bool

Errored returns whether the buffer contains an error.

func (*ReadBuffer) Fill

func (buffer *ReadBuffer) Fill(data []byte)

Fill fills the buffer with the specified bytes.

func (*ReadBuffer) IncludeError

func (buffer *ReadBuffer) IncludeError(err error)

IncludeError adds error at the end of the buffer.

func (*ReadBuffer) Read

func (buffer *ReadBuffer) Read(data []byte) (n int, err error)

Read reads from the buffer.