storj

package
v0.13.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 11, 2019 License: AGPL-3.0 Imports: 19 Imported by: 0

Documentation

Overview

Package storj contains the types which represent the main entities of the Storj domain.

Index

Constants

View Source 
const (
	// Unencrypted indicates no encryption or decryption is to be performed.
	Unencrypted = Cipher(iota)
	// AESGCM indicates use of AES128-GCM encryption.
	AESGCM
	// SecretBox indicates use of XSalsa20-Poly1305 encryption, as provided by
	// the NaCl cryptography library under the name "Secretbox".
	SecretBox
	// Invalid indicates a Cipher value whose use is not valid. This may be
	// used as a replacement for "unspecified" in a pinch, although it is not
	// the zero value.
	Invalid
)

List of supported encryption algorithms

View Source 
const (
	// EncUnspecified indicates no encryption suite has been selected.
	EncUnspecified = CipherSuite(iota)
	// EncNull indicates use of the NULL cipher; that is, no encryption is
	// done. The ciphertext is equal to the plaintext.
	EncNull
	// EncAESGCM indicates use of AES128-GCM encryption.
	EncAESGCM
	// EncSecretBox indicates use of XSalsa20-Poly1305 encryption, as provided
	// by the NaCl cryptography library under the name "Secretbox".
	EncSecretBox
)
View Source 
const (
	KeySize   = 32
	NonceSize = 24
)

Constant definitions for key and nonce sizes

View Source 
const (
	// Before lists backwards from cursor, without cursor
	Before = ListDirection(-2)
	// Backward lists backwards from cursor, including cursor
	Backward = ListDirection(-1)
	// Forward lists forwards from cursor, including cursor
	Forward = ListDirection(1)
	// After lists forwards from cursor, without cursor
	After = ListDirection(2)
)
View Source 
const (
	InvalidRedundancyAlgorithm = RedundancyAlgorithm(iota)
	ReedSolomon
)

List of supported redundancy algorithms

View Source 
const NodeIDSize = sha256.Size

NodeIDSize is the byte length of a NodeID

View Source 
const (
	// V0 represents identity version 0
	// NB: identities created before identity versioning (i.e. which don't have a
	// version extension; "legacy") will be recognized as V0.
	V0 = IDVersionNumber(iota)
)

Variables

View Source 
var (
	// IDVersions is a map of all identity versions
	IDVersions = map[IDVersionNumber]IDVersion{
		V0: {
			Number:        V0,
			NewPrivateKey: pkcrypto.GeneratePrivateKey,
		},
	}

	// IDVersionHandler compares the identity version of the remote peers
	// certificate chain to the extension options passed to the factory.
	IDVersionHandler = extensions.NewHandlerFactory(
		&extensions.IdentityVersionExtID, idVersionHandler,
	)
)
View Source 
var (
	// ErrNodeID is used when something goes wrong with a node id.
	ErrNodeID = errs.Class("node ID error")
	// ErrVersion is used for identity version related errors.
	ErrVersion = errs.Class("node ID version error")
)
View Source 
var (
	// ErrNoBucket is an error class for using empty bucket name
	ErrNoBucket = errs.Class("no bucket specified")

	// ErrNoPath is an error class for using empty path
	ErrNoPath = errs.Class("no path specified")

	// ErrBucketNotFound is an error class for non-existing bucket
	ErrBucketNotFound = errs.Class("bucket not found")

	// ErrObjectNotFound is an error class for non-existing object
	ErrObjectNotFound = errs.Class("object not found")
)
View Source 
var ErrPieceID = errs.Class("piece ID error")

ErrPieceID is used when something goes wrong with a piece ID

View Source 
var ErrSerialNumber = errs.Class("serial number error")

ErrSerialNumber is used when something goes wrong with a serial number

Functions

func IDVersionInVersions added in v0.9.0 

func IDVersionInVersions(versionNumber IDVersionNumber, versionsStr string) error

IDVersionInVersions returns an error if the given version is in the given string of version(s)/range(s).

func NewVersionExt added in v0.9.0 

func NewVersionExt(version IDVersion) pkix.Extension

NewVersionExt creates a new identity version certificate extension for the given identity version,

func SplitPath 

func SplitPath(path Path) []string

SplitPath splits path into a slice of path components

Types

type Bucket 

type Bucket struct {
	Name                 string
	Created              time.Time
	PathCipher           Cipher
	SegmentsSize         int64
	RedundancyScheme     RedundancyScheme
	EncryptionParameters EncryptionParameters
}

Bucket contains information about a specific bucket

type BucketList 

type BucketList struct {
	More  bool
	Items []Bucket
}

BucketList is a list of buckets

type BucketListOptions 

type BucketListOptions struct {
	Cursor    string
	Direction ListDirection
	Limit     int
}

BucketListOptions lists objects

func (BucketListOptions) NextPage 

func (opts BucketListOptions) NextPage(list BucketList) BucketListOptions

NextPage returns options for listing the next page

type Cipher 

type Cipher byte

Cipher specifies an encryption algorithm

func (Cipher) ToCipherSuite 

func (c Cipher) ToCipherSuite() CipherSuite

ToCipherSuite converts a Cipher value to a CipherSuite value.

type CipherSuite 

type CipherSuite byte

CipherSuite specifies one of the encryption suites supported by Storj libraries for encryption of in-network data.

func (CipherSuite) ToCipher 

func (cs CipherSuite) ToCipher() Cipher

ToCipher converts a CipherSuite value to a Cipher value.

type CreateObject 

type CreateObject struct {
	Metadata    map[string]string
	ContentType string
	Expires     time.Time

	RedundancyScheme
	EncryptionScheme
}

CreateObject has optional parameters that can be set

func (CreateObject) Object 

func (create CreateObject) Object(bucket Bucket, path Path) Object

Object converts the CreateObject to an object with unitialized values

type EncryptedPrivateKey 

type EncryptedPrivateKey []byte

EncryptedPrivateKey is a private key that has been encrypted

type EncryptionParameters 

type EncryptionParameters struct {
	// CipherSuite specifies the cipher suite to be used for encryption.
	CipherSuite CipherSuite
	// BlockSize determines the unit size at which encryption is performed.
	// It is important to distinguish this from the block size used by the
	// cipher suite (probably 128 bits). There is some small overhead for
	// each encryption unit, so BlockSize should not be too small, but
	// smaller sizes yield shorter first-byte latency and better seek times.
	// Note that BlockSize itself is the size of data blocks _after_ they
	// have been encrypted and the authentication overhead has been added.
	// It is _not_ the size of the data blocks to _be_ encrypted.
	BlockSize int32
}

EncryptionParameters is the cipher suite and parameters used for encryption It is like EncryptionScheme, but uses the CipherSuite type instead of Cipher. EncryptionParameters is preferred for new uses.

func (EncryptionParameters) IsZero 

func (params EncryptionParameters) IsZero() bool

IsZero returns true if no field in the struct is set to non-zero value

func (EncryptionParameters) ToEncryptionScheme 

func (params EncryptionParameters) ToEncryptionScheme() EncryptionScheme

ToEncryptionScheme transforms an EncryptionParameters object into the equivalent EncryptionScheme object.

type EncryptionScheme 

type EncryptionScheme struct {
	// Cipher specifies the cipher suite to be used for encryption.
	Cipher Cipher
	// BlockSize determines the unit size at which encryption is performed.
	// It is important to distinguish this from the block size used by the
	// cipher suite (probably 128 bits). There is some small overhead for
	// each encryption unit, so BlockSize should not be too small, but
	// smaller sizes yield shorter first-byte latency and better seek times.
	// Note that BlockSize itself is the size of data blocks _after_ they
	// have been encrypted and the authentication overhead has been added.
	// It is _not_ the size of the data blocks to _be_ encrypted.
	BlockSize int32
}

EncryptionScheme is the scheme and parameters used for encryption. Use the similar EncryptionParameters struct instead, if possible.

func (EncryptionScheme) IsZero 

func (scheme EncryptionScheme) IsZero() bool

IsZero returns true if no field in the struct is set to non-zero value

func (EncryptionScheme) ToEncryptionParameters 

func (scheme EncryptionScheme) ToEncryptionParameters() EncryptionParameters

ToEncryptionParameters transforms an EncryptionScheme object into the equivalent EncryptionParameters object.

type IDVersion 

type IDVersion struct {
	Number        IDVersionNumber
	NewPrivateKey func() (crypto.PrivateKey, error)
}

IDVersion holds fields that are used to distinguish different identity versions from one another; used in identity generation.

func GetIDVersion added in v0.9.0 

func GetIDVersion(number IDVersionNumber) (IDVersion, error)

GetIDVersion looks up the given version number in the map of registered versions, returning an error if none is found.

func IDVersionFromCert added in v0.9.0 

func IDVersionFromCert(cert *x509.Certificate) (IDVersion, error)

IDVersionFromCert parsed the IDVersion from the passed certificate's IDVersion extension.

func LatestIDVersion added in v0.9.0 

func LatestIDVersion() IDVersion

LatestIDVersion returns the last IDVersion registered.

type IDVersionNumber added in v0.9.0 

type IDVersionNumber uint8

IDVersionNumber is the number of an identity version.

type Key 

type Key [KeySize]byte

Key represents the largest key used by any encryption protocol

func NewKey added in v0.12.0 

func NewKey(humanReadableKey []byte) (*Key, error)

NewKey creates a new Storj key from humanReadableKey.

func (*Key) IsZero added in v0.12.0 

func (key *Key) IsZero() bool

IsZero returns true if key is nil or it points to its zero value

func (*Key) Raw 

func (key *Key) Raw() *[KeySize]byte

Raw returns the key as a raw byte array pointer

type LastSegment 

type LastSegment struct {
	Size              int64
	EncryptedKeyNonce Nonce
	EncryptedKey      EncryptedPrivateKey
}

LastSegment contains info about last segment TODO: remove

type ListDirection 

type ListDirection int8

ListDirection specifies listing direction

type ListOptions 

type ListOptions struct {
	Prefix    Path
	Cursor    Path // Cursor is relative to Prefix, full path is Prefix + Cursor
	Delimiter rune
	Recursive bool
	Direction ListDirection
	Limit     int
}

ListOptions lists objects

func (ListOptions) NextPage 

func (opts ListOptions) NextPage(list ObjectList) ListOptions

NextPage returns options for listing the next page

type Metainfo 

type Metainfo interface {
	// Limits returns limits for this metainfo database
	Limits() (MetainfoLimits, error)

	// CreateBucket creates a new bucket with the specified information
	// Database automatically sets different values in the information
	CreateBucket(ctx context.Context, bucket string, info *Bucket) (Bucket, error)
	// DeleteBucket deletes bucket
	DeleteBucket(ctx context.Context, bucket string) error
	// GetBucket gets bucket information
	GetBucket(ctx context.Context, bucket string) (Bucket, error)
	// ListBuckets lists buckets starting from first
	ListBuckets(ctx context.Context, options BucketListOptions) (BucketList, error)

	// GetObject returns information about an object
	GetObject(ctx context.Context, bucket string, path Path) (Object, error)
	// GetObjectStream returns interface for reading the object stream
	GetObjectStream(ctx context.Context, bucket string, path Path) (ReadOnlyStream, error)

	// CreateObject creates a mutable object for uploading stream info
	CreateObject(ctx context.Context, bucket string, path Path, info *CreateObject) (MutableObject, error)
	// ModifyObject creates a mutable object for updating a partially uploaded object
	ModifyObject(ctx context.Context, bucket string, path Path) (MutableObject, error)
	// DeleteObject deletes an object from database
	DeleteObject(ctx context.Context, bucket string, path Path) error
	// ListObjects lists objects in bucket based on the ListOptions
	ListObjects(ctx context.Context, bucket string, options ListOptions) (ObjectList, error)

	// ModifyPendingObject creates a mutable object for updating a partially uploaded object
	ModifyPendingObject(ctx context.Context, bucket string, path Path) (MutableObject, error)
	// ListPendingObjects lists pending objects in bucket based on the ListOptions
	ListPendingObjects(ctx context.Context, bucket string, options ListOptions) (ObjectList, error)
}

Metainfo represents a database for storing meta-info about objects

type MetainfoLimits 

type MetainfoLimits struct {
	// ListLimit specifies the maximum amount of items that can be listed at a time.
	ListLimit int64

	// MinimumRemoteSegmentSize specifies the minimum remote segment that is allowed to be stored.
	MinimumRemoteSegmentSize int64
	// MaximumInlineSegmentSize specifies the maximum inline segment that is allowed to be stored.
	MaximumInlineSegmentSize int64
}

MetainfoLimits lists limits specified for the Metainfo database

type MutableObject 

type MutableObject interface {
	// Info gets the current information about the object
	Info() Object

	// CreateStream creates a new stream for the object
	CreateStream(ctx context.Context) (MutableStream, error)
	// ContinueStream starts to continue a partially uploaded stream.
	ContinueStream(ctx context.Context) (MutableStream, error)
	// DeleteStream deletes any information about this objects stream
	DeleteStream(ctx context.Context) error

	// Commit commits the changes to the database
	Commit(ctx context.Context) error
}

MutableObject is an interface for manipulating creating/deleting object stream

type MutableStream 

type MutableStream interface {
	Info() Object
	// AddSegments adds segments to the stream.
	AddSegments(ctx context.Context, segments ...Segment) error
	// UpdateSegments updates information about segments.
	UpdateSegments(ctx context.Context, segments ...Segment) error
}

MutableStream is an interface for manipulating stream information

type NodeID 

type NodeID [NodeIDSize]byte

NodeID is a unique node identifier

func NewVersionedID added in v0.9.0 

func NewVersionedID(id NodeID, version IDVersion) NodeID

NewVersionedID adds an identity version to a node ID.

func NodeIDFromBytes 

func NodeIDFromBytes(b []byte) (NodeID, error)

NodeIDFromBytes converts a byte slice into a node id

func NodeIDFromString 

func NodeIDFromString(s string) (NodeID, error)

NodeIDFromString decodes a base58check encoded node id string

func (NodeID) Bytes 

func (id NodeID) Bytes() []byte

Bytes returns raw bytes of the id

func (NodeID) Difficulty 

func (id NodeID) Difficulty() (uint16, error)

Difficulty returns the number of trailing zero bits in a node ID

func (NodeID) IsZero 

func (id NodeID) IsZero() bool

IsZero returns whether NodeID is unassigned

func (NodeID) Less 

func (id NodeID) Less(b NodeID) bool

Less returns whether id is smaller than b in lexiographic order

func (NodeID) Marshal 

func (id NodeID) Marshal() ([]byte, error)

Marshal serializes a node id

func (NodeID) MarshalJSON 

func (id NodeID) MarshalJSON() ([]byte, error)

MarshalJSON serializes a node ID to a json string as bytes

func (*NodeID) MarshalTo 

func (id *NodeID) MarshalTo(data []byte) (n int, err error)

MarshalTo serializes a node ID into the passed byte slice

func (*NodeID) Scan 

func (id *NodeID) Scan(src interface{}) (err error)

Scan extracts a NodeID from a database field

func (*NodeID) Size 

func (id *NodeID) Size() int

Size returns the length of a node ID (implements gogo's custom type interface)

func (NodeID) String 

func (id NodeID) String() string

String returns NodeID as base58 encoded string with checksum and version bytes

func (*NodeID) Unmarshal 

func (id *NodeID) Unmarshal(data []byte) error

Unmarshal deserializes a node ID

func (*NodeID) UnmarshalJSON 

func (id *NodeID) UnmarshalJSON(data []byte) error

UnmarshalJSON deserializes a json string (as bytes) to a node ID

func (NodeID) Value 

func (id NodeID) Value() (driver.Value, error)

Value converts a NodeID to a database field

func (NodeID) Version added in v0.9.0 

func (id NodeID) Version() IDVersion

Version returns the version of the identity format

type NodeIDList 

type NodeIDList []NodeID

NodeIDList is a slice of NodeIDs (implements sort)

func NodeIDsFromBytes 

func NodeIDsFromBytes(b [][]byte) (ids NodeIDList, err error)

NodeIDsFromBytes converts a 2d byte slice into a list of nodes

func (NodeIDList) Bytes 

func (n NodeIDList) Bytes() (idsBytes [][]byte)

Bytes returns a 2d byte slice of the node IDs

func (NodeIDList) Len 

func (n NodeIDList) Len() int

Len implements sort.Interface.Len()

func (NodeIDList) Less 

func (n NodeIDList) Less(i, j int) bool

Less implements sort.Interface.Less()

func (NodeIDList) Strings added in v0.13.0 

func (n NodeIDList) Strings() []string

Strings returns a string slice of the node IDs

func (NodeIDList) Swap 

func (n NodeIDList) Swap(i, j int)

Swap implements sort.Interface.Swap()

type Nonce 

type Nonce [NonceSize]byte

Nonce represents the largest nonce used by any encryption protocol

func (*Nonce) Raw 

func (nonce *Nonce) Raw() *[NonceSize]byte

Raw returns the nonce as a raw byte array pointer

type Object 

type Object struct {
	Version  uint32
	Bucket   Bucket
	Path     Path
	IsPrefix bool

	Metadata map[string]string

	ContentType string
	Created     time.Time
	Modified    time.Time
	Expires     time.Time

	Stream
}

Object contains information about a specific object

type ObjectList 

type ObjectList struct {
	Bucket string
	Prefix Path
	More   bool

	// Items paths are relative to Prefix
	// To get the full path use list.Prefix + list.Items[0].Path
	Items []Object
}

ObjectList is a list of objects

type Path 

type Path = string

Path represents a object path

func JoinPaths 

func JoinPaths(paths ...Path) Path

JoinPaths concatenates paths to a new single path

type Piece 

type Piece struct {
	Number   byte
	Location NodeID
}

Piece is information where a piece is located

type PieceID 

type PieceID [32]byte

PieceID is the unique identifier for pieces

func NewPieceID 

func NewPieceID() PieceID

NewPieceID creates a piece ID

func PieceIDFromBytes 

func PieceIDFromBytes(b []byte) (PieceID, error)

PieceIDFromBytes converts a byte slice into a piece ID

func PieceIDFromString 

func PieceIDFromString(s string) (PieceID, error)

PieceIDFromString decodes a hex encoded piece ID string

func (PieceID) Bytes 

func (id PieceID) Bytes() []byte

Bytes returns bytes of the piece ID

func (PieceID) Derive 

func (id PieceID) Derive(storagenodeID NodeID) PieceID

Derive a new PieceID from the current piece ID and the given storage node ID

func (PieceID) IsZero 

func (id PieceID) IsZero() bool

IsZero returns whether piece ID is unassigned

func (PieceID) Marshal 

func (id PieceID) Marshal() ([]byte, error)

Marshal serializes a piece ID

func (PieceID) MarshalJSON 

func (id PieceID) MarshalJSON() ([]byte, error)

MarshalJSON serializes a piece ID to a json string as bytes

func (*PieceID) MarshalTo 

func (id *PieceID) MarshalTo(data []byte) (n int, err error)

MarshalTo serializes a piece ID into the passed byte slice

func (*PieceID) Scan 

func (id *PieceID) Scan(src interface{}) (err error)

Scan extracts a PieceID from a database field

func (*PieceID) Size 

func (id *PieceID) Size() int

Size returns the length of a piece ID (implements gogo's custom type interface)

func (PieceID) String 

func (id PieceID) String() string

String representation of the piece ID

func (*PieceID) Unmarshal 

func (id *PieceID) Unmarshal(data []byte) error

Unmarshal deserializes a piece ID

func (*PieceID) UnmarshalJSON 

func (id *PieceID) UnmarshalJSON(data []byte) error

UnmarshalJSON deserializes a json string (as bytes) to a piece ID

func (PieceID) Value 

func (id PieceID) Value() (driver.Value, error)

Value set a PieceID to a database field

type ReadOnlyStream 

type ReadOnlyStream interface {
	Info() Object

	// SegmentsAt returns the segment that contains the byteOffset and following segments.
	// Limit specifies how much to return at most.
	SegmentsAt(ctx context.Context, byteOffset int64, limit int64) (infos []Segment, more bool, err error)
	// Segments returns the segment at index.
	// Limit specifies how much to return at most.
	Segments(ctx context.Context, index int64, limit int64) (infos []Segment, more bool, err error)
}

ReadOnlyStream is an interface for reading segment information

type RedundancyAlgorithm 

type RedundancyAlgorithm byte

RedundancyAlgorithm is the algorithm used for redundancy

type RedundancyScheme 

type RedundancyScheme struct {
	// Algorithm determines the algorithm to be used for redundancy.
	Algorithm RedundancyAlgorithm

	// ShareSize is the size in bytes for each erasure shares.
	ShareSize int32

	// RequiredShares is the minimum number of shares required to recover a
	// stripe, reed-solomon k.
	RequiredShares int16
	// RepairShares is the minimum number of safe shares that can remain
	// before a repair is triggered.
	RepairShares int16
	// OptimalShares is the desired total number of shares for a segment.
	OptimalShares int16
	// TotalShares is the number of shares to encode. If it is larger than
	// OptimalShares, slower uploads of the excess shares will be aborted in
	// order to improve performance.
	TotalShares int16
}

RedundancyScheme specifies the parameters and the algorithm for redundancy

func (RedundancyScheme) IsZero 

func (scheme RedundancyScheme) IsZero() bool

IsZero returns true if no field in the struct is set to non-zero value

func (RedundancyScheme) StripeSize added in v0.13.0 

func (scheme RedundancyScheme) StripeSize() int32

StripeSize is the number of bytes for a stripe. Stripes are erasure encoded and split into n shares, where we need k to reconstruct the stripe. Therefore a stripe size is the erasure share size times the required shares, k.

type Segment 

type Segment struct {
	Index int64
	// Size is the size of the content in bytes
	Size int64
	// Checksum is the checksum of the content
	Checksum []byte
	// Local data
	Inline []byte
	// Remote data
	PieceID PieceID
	Pieces  []Piece
	// Encryption
	EncryptedKeyNonce Nonce
	EncryptedKey      EncryptedPrivateKey
}

Segment is full segment information

type SerialNumber 

type SerialNumber [16]byte

SerialNumber is the unique identifier for pieces

func SerialNumberFromBytes 

func SerialNumberFromBytes(b []byte) (SerialNumber, error)

SerialNumberFromBytes converts a byte slice into a serial number

func SerialNumberFromString 

func SerialNumberFromString(s string) (SerialNumber, error)

SerialNumberFromString decodes an base32 encoded

func (SerialNumber) Bytes 

func (id SerialNumber) Bytes() []byte

Bytes returns bytes of the serial number

func (SerialNumber) IsZero 

func (id SerialNumber) IsZero() bool

IsZero returns whether serial number is unassigned

func (SerialNumber) Marshal 

func (id SerialNumber) Marshal() ([]byte, error)

Marshal serializes a serial number

func (SerialNumber) MarshalJSON 

func (id SerialNumber) MarshalJSON() ([]byte, error)

MarshalJSON serializes a serial number to a json string as bytes

func (*SerialNumber) MarshalTo 

func (id *SerialNumber) MarshalTo(data []byte) (n int, err error)

MarshalTo serializes a serial number into the passed byte slice

func (*SerialNumber) Scan 

func (id *SerialNumber) Scan(src interface{}) (err error)

Scan extracts a SerialNumber from a database field

func (*SerialNumber) Size 

func (id *SerialNumber) Size() int

Size returns the length of a serial number (implements gogo's custom type interface)

func (SerialNumber) String 

func (id SerialNumber) String() string

String representation of the serial number

func (*SerialNumber) Unmarshal 

func (id *SerialNumber) Unmarshal(data []byte) error

Unmarshal deserializes a serial number

func (*SerialNumber) UnmarshalJSON 

func (id *SerialNumber) UnmarshalJSON(data []byte) error

UnmarshalJSON deserializes a json string (as bytes) to a serial number

func (SerialNumber) Value 

func (id SerialNumber) Value() (driver.Value, error)

Value set a SerialNumber to a database field

type Stream 

type Stream struct {
	// Size is the total size of the stream in bytes
	Size int64
	// Checksum is the checksum of the segment checksums
	Checksum []byte

	// SegmentCount is the number of segments
	SegmentCount int64
	// FixedSegmentSize is the size of each segment,
	// when all segments have the same size. It is -1 otherwise.
	FixedSegmentSize int64

	// RedundancyScheme specifies redundancy strategy used for this stream
	RedundancyScheme
	// EncryptionScheme specifies encryption strategy used for this stream
	EncryptionScheme

	LastSegment LastSegment // TODO: remove
}

Stream is information about an object stream

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.