metabase

package
v1.27.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 14, 2021 License: AGPL-3.0 Imports: 30 Imported by: 0

Documentation

Overview

Package metabase implements storing objects and segements.

Index

Constants

View Source 
const (
	Delimiter        = '/'
	LastSegmentName  = "l"
	LastSegmentIndex = uint32(math.MaxUint32)
)

Common constants for segment keys.

View Source 
const (
	// Pending means that the object is being uploaded or that the client failed during upload.
	// The failed upload may be continued in the future.
	Pending = ObjectStatus(1)
	// Committed means that the object is finished and should be visible for general listing.
	Committed = ObjectStatus(3)
)
View Source 
const MaxListLimit = 1000

MaxListLimit is the maximum number of items the client can request for listing.

View Source 
const NextVersion = Version(0)

NextVersion means that the version should be chosen automatically.

Variables

View Source 
var (
	// ErrInvalidRequest is used to indicate invalid requests.
	ErrInvalidRequest = errs.Class("metabase: invalid request")
	// ErrConflict is used to indicate conflict with the request.
	ErrConflict = errs.Class("metabase: conflict")
)
View Source 
var ErrSegmentNotFound = errs.Class("segment not found")

ErrSegmentNotFound is an error class for non-existing segment.

View Source 
var Error = errs.Class("metabase")

Error is the default error for metabase.

Functions

func EqualAliasPieces added in v1.26.2 

func EqualAliasPieces(xs, ys AliasPieces) bool

EqualAliasPieces compares whether xs and ys are equal.

Types

type AliasPiece added in v1.26.2 

type AliasPiece struct {
	Number uint16
	Alias  NodeAlias
}

AliasPiece is a piece with alias node ID.

type AliasPieces added in v1.26.2 

type AliasPieces []AliasPiece

AliasPieces is a slice of AliasPiece.

func (AliasPieces) Bytes added in v1.26.2 

func (aliases AliasPieces) Bytes() ([]byte, error)

Bytes compresses alias pieces to a slice of bytes.

func (*AliasPieces) Scan added in v1.26.2 

func (aliases *AliasPieces) Scan(src interface{}) error

Scan implements the database/sql Scanner interface.

func (*AliasPieces) SetBytes added in v1.26.2 

func (aliases *AliasPieces) SetBytes(data []byte) error

SetBytes decompresses alias pieces from a slice of bytes.

func (AliasPieces) Value added in v1.26.2 

func (aliases AliasPieces) Value() (driver.Value, error)

Value implements the database/sql/driver Valuer interface.

type BeginObjectExactVersion added in v1.26.2 

type BeginObjectExactVersion struct {
	ObjectStream

	ExpiresAt              *time.Time
	ZombieDeletionDeadline *time.Time

	Encryption storj.EncryptionParameters
}

BeginObjectExactVersion contains arguments necessary for starting an object upload.

type BeginObjectNextVersion added in v1.26.2 

type BeginObjectNextVersion struct {
	ObjectStream

	ExpiresAt              *time.Time
	ZombieDeletionDeadline *time.Time

	Encryption storj.EncryptionParameters
}

BeginObjectNextVersion contains arguments necessary for starting an object upload.

type BeginSegment added in v1.26.2 

type BeginSegment struct {
	ObjectStream

	Position    SegmentPosition
	RootPieceID storj.PieceID
	Pieces      Pieces
}

BeginSegment contains options to verify, whether a new segment upload can be started.

type BucketEmpty added in v1.26.2 

type BucketEmpty struct {
	ProjectID  uuid.UUID
	BucketName string
}

BucketEmpty contains arguments necessary for checking if bucket is empty.

type BucketLocation 

type BucketLocation struct {
	ProjectID  uuid.UUID
	BucketName string
}

BucketLocation defines a bucket that belongs to a project.

func ParseBucketPrefix 

func ParseBucketPrefix(prefix BucketPrefix) (BucketLocation, error)

ParseBucketPrefix parses BucketPrefix.

func ParseCompactBucketPrefix added in v1.19.9 

func ParseCompactBucketPrefix(compactPrefix []byte) (BucketLocation, error)

ParseCompactBucketPrefix parses BucketPrefix.

func (BucketLocation) CompactPrefix added in v1.19.9 

func (loc BucketLocation) CompactPrefix() []byte

CompactPrefix converts bucket location into bucket prefix with compact project ID.

func (BucketLocation) Prefix 

func (loc BucketLocation) Prefix() BucketPrefix

Prefix converts bucket location into bucket prefix.

func (BucketLocation) Verify added in v1.26.2 

func (loc BucketLocation) Verify() error

Verify object location fields.

type BucketPrefix 

type BucketPrefix string

BucketPrefix consists of <project id>/<bucket name>.

type CommitInlineSegment added in v1.26.2 

type CommitInlineSegment struct {
	ObjectStream

	Position SegmentPosition

	EncryptedKeyNonce []byte
	EncryptedKey      []byte

	PlainOffset   int64 // offset in the original data stream
	PlainSize     int32 // size before encryption
	EncryptedETag []byte

	InlineData []byte
}

CommitInlineSegment contains all necessary information about the segment.

type CommitObject added in v1.26.2 

type CommitObject struct {
	ObjectStream

	Encryption storj.EncryptionParameters

	EncryptedMetadata             []byte
	EncryptedMetadataNonce        []byte
	EncryptedMetadataEncryptedKey []byte
}

CommitObject contains arguments necessary for committing an object.

type CommitObjectWithSegments added in v1.26.2 

type CommitObjectWithSegments struct {
	ObjectStream

	EncryptedMetadata             []byte
	EncryptedMetadataNonce        []byte
	EncryptedMetadataEncryptedKey []byte

	// TODO: this probably should use segment ranges rather than individual items
	Segments []SegmentPosition
}

CommitObjectWithSegments contains arguments necessary for committing an object.

type CommitSegment added in v1.26.2 

type CommitSegment struct {
	ObjectStream

	Position    SegmentPosition
	RootPieceID storj.PieceID

	EncryptedKeyNonce []byte
	EncryptedKey      []byte

	PlainOffset   int64 // offset in the original data stream
	PlainSize     int32 // size before encryption
	EncryptedSize int32 // segment size after encryption

	EncryptedETag []byte

	Redundancy storj.RedundancyScheme

	Pieces Pieces
}

CommitSegment contains all necessary information about the segment.

type DB added in v1.26.2 

type DB struct {
	// contains filtered or unexported fields
}

DB implements a database for storing objects and segments.

func Open added in v1.26.2 

func Open(ctx context.Context, log *zap.Logger, driverName, connstr string) (*DB, error)

Open opens a connection to metabase.

func (*DB) BeginObjectExactVersion added in v1.26.2 

func (db *DB) BeginObjectExactVersion(ctx context.Context, opts BeginObjectExactVersion) (committed Object, err error)

BeginObjectExactVersion adds a pending object to the database, with specific version.

func (*DB) BeginObjectNextVersion added in v1.26.2 

func (db *DB) BeginObjectNextVersion(ctx context.Context, opts BeginObjectNextVersion) (committed Version, err error)

BeginObjectNextVersion adds a pending object to the database, with automatically assigned version.

func (*DB) BeginSegment added in v1.26.2 

func (db *DB) BeginSegment(ctx context.Context, opts BeginSegment) (err error)

BeginSegment verifies, whether a new segment upload can be started.

func (*DB) BucketEmpty added in v1.26.2 

func (db *DB) BucketEmpty(ctx context.Context, opts BucketEmpty) (empty bool, err error)

BucketEmpty returns true if bucket does not contain objects (pending or committed). This method doesn't check bucket existence.

func (*DB) CheckVersion added in v1.26.2 

func (db *DB) CheckVersion(ctx context.Context) error

CheckVersion checks the database is the correct version.

func (*DB) Close added in v1.26.2 

func (db *DB) Close() error

Close closes the connection to database.

func (*DB) CommitInlineSegment added in v1.26.2 

func (db *DB) CommitInlineSegment(ctx context.Context, opts CommitInlineSegment) (err error)

CommitInlineSegment commits inline segment to the database.

func (*DB) CommitObject added in v1.26.2 

func (db *DB) CommitObject(ctx context.Context, opts CommitObject) (object Object, err error)

CommitObject adds a pending object to the database.

func (*DB) CommitObjectWithSegments added in v1.26.2 

func (db *DB) CommitObjectWithSegments(ctx context.Context, opts CommitObjectWithSegments) (object Object, deletedSegments []DeletedSegmentInfo, err error)

CommitObjectWithSegments commits pending object to the database.

func (*DB) CommitSegment added in v1.26.2 

func (db *DB) CommitSegment(ctx context.Context, opts CommitSegment) (err error)

CommitSegment commits segment to the database.

func (*DB) DeleteBucketObjects added in v1.26.2 

func (db *DB) DeleteBucketObjects(ctx context.Context, opts DeleteBucketObjects) (deletedObjectCount int64, err error)

DeleteBucketObjects deletes all objects in the specified bucket.

func (*DB) DeleteExpiredObjects added in v1.26.2 

func (db *DB) DeleteExpiredObjects(ctx context.Context, opts DeleteExpiredObjects) (err error)

DeleteExpiredObjects deletes all objects that expired before expiredBefore.

func (*DB) DeleteObjectAnyStatusAllVersions added in v1.26.2 

func (db *DB) DeleteObjectAnyStatusAllVersions(ctx context.Context, opts DeleteObjectAnyStatusAllVersions) (result DeleteObjectResult, err error)

DeleteObjectAnyStatusAllVersions deletes all object versions.

func (*DB) DeleteObjectExactVersion added in v1.26.2 

func (db *DB) DeleteObjectExactVersion(ctx context.Context, opts DeleteObjectExactVersion) (result DeleteObjectResult, err error)

DeleteObjectExactVersion deletes an exact object version.

func (*DB) DeleteObjectLatestVersion added in v1.26.2 

func (db *DB) DeleteObjectLatestVersion(ctx context.Context, opts DeleteObjectLatestVersion) (result DeleteObjectResult, err error)

DeleteObjectLatestVersion deletes latest object version.

func (*DB) DeleteObjectsAllVersions added in v1.26.2 

func (db *DB) DeleteObjectsAllVersions(ctx context.Context, opts DeleteObjectsAllVersions) (result DeleteObjectResult, err error)

DeleteObjectsAllVersions deletes all versions of multiple objects from the same bucket.

func (*DB) DeletePendingObject added in v1.26.2 

func (db *DB) DeletePendingObject(ctx context.Context, opts DeletePendingObject) (result DeleteObjectResult, err error)

DeletePendingObject deletes a pending object with specified version and streamID.

func (*DB) DestroyTables added in v1.26.2 

func (db *DB) DestroyTables(ctx context.Context) error

DestroyTables deletes all tables.

TODO: remove this, only for bootstrapping.

func (*DB) EnsureNodeAliases added in v1.26.2 

func (db *DB) EnsureNodeAliases(ctx context.Context, opts EnsureNodeAliases) (err error)

EnsureNodeAliases ensures that the supplied node ID-s have a alias. It's safe to concurrently try and create node ID-s for the same NodeID.

func (*DB) GetLatestObjectLastSegment added in v1.26.2 

func (db *DB) GetLatestObjectLastSegment(ctx context.Context, opts GetLatestObjectLastSegment) (segment Segment, err error)

GetLatestObjectLastSegment returns an object last segment information.

func (*DB) GetObjectExactVersion added in v1.26.2 

func (db *DB) GetObjectExactVersion(ctx context.Context, opts GetObjectExactVersion) (_ Object, err error)

GetObjectExactVersion returns object information for exact version.

func (*DB) GetObjectLatestVersion added in v1.26.2 

func (db *DB) GetObjectLatestVersion(ctx context.Context, opts GetObjectLatestVersion) (_ Object, err error)

GetObjectLatestVersion returns object information for latest version.

func (*DB) GetSegmentByLocation added in v1.26.2 

func (db *DB) GetSegmentByLocation(ctx context.Context, opts GetSegmentByLocation) (segment Segment, err error)

GetSegmentByLocation returns information about segment on the specified location.

func (*DB) GetSegmentByOffset added in v1.26.2 

func (db *DB) GetSegmentByOffset(ctx context.Context, opts GetSegmentByOffset) (segment Segment, err error)

GetSegmentByOffset returns an object segment information.

func (*DB) GetSegmentByPosition added in v1.26.2 

func (db *DB) GetSegmentByPosition(ctx context.Context, opts GetSegmentByPosition) (segment Segment, err error)

GetSegmentByPosition returns information about segment on the specified position.

func (*DB) GetStreamPieceCountByNodeID added in v1.26.2 

func (db *DB) GetStreamPieceCountByNodeID(ctx context.Context, opts GetStreamPieceCountByNodeID) (result map[storj.NodeID]int64, err error)

GetStreamPieceCountByNodeID returns piece count by node id.

func (*DB) InternalImplementation added in v1.26.2 

func (db *DB) InternalImplementation() interface{}

InternalImplementation returns *metabase.DB. TODO: remove.

func (*DB) IterateLoopObjects added in v1.26.2 

func (db *DB) IterateLoopObjects(ctx context.Context, opts IterateLoopObjects, fn func(context.Context, LoopObjectsIterator) error) (err error)

IterateLoopObjects iterates through all objects in metabase.

func (*DB) IterateLoopStreams added in v1.26.2 

func (db *DB) IterateLoopStreams(ctx context.Context, opts IterateLoopStreams, handleStream func(ctx context.Context, streamID uuid.UUID, next SegmentIterator) error) (err error)

IterateLoopStreams lists multiple streams segments.

func (*DB) IterateObjectsAllVersions added in v1.26.2 

func (db *DB) IterateObjectsAllVersions(ctx context.Context, opts IterateObjects, fn func(context.Context, ObjectsIterator) error) (err error)

IterateObjectsAllVersions iterates through all versions of all objects.

func (*DB) IterateObjectsAllVersionsWithStatus added in v1.26.2 

func (db *DB) IterateObjectsAllVersionsWithStatus(ctx context.Context, opts IterateObjectsWithStatus, fn func(context.Context, ObjectsIterator) error) (err error)

IterateObjectsAllVersionsWithStatus iterates through all versions of all objects with specified status.

func (*DB) IteratePendingObjectsByKey added in v1.26.2 

func (db *DB) IteratePendingObjectsByKey(ctx context.Context, opts IteratePendingObjectsByKey, fn func(context.Context, ObjectsIterator) error) (err error)

IteratePendingObjectsByKey iterates through all streams of pending objects with the same ObjectKey.

func (*DB) ListNodeAliases added in v1.26.2 

func (db *DB) ListNodeAliases(ctx context.Context) (_ []NodeAliasEntry, err error)

ListNodeAliases lists all node alias mappings.

func (*DB) ListSegments added in v1.26.2 

func (db *DB) ListSegments(ctx context.Context, opts ListSegments) (result ListSegmentsResult, err error)

ListSegments lists specified stream segments.

func (*DB) ListStreamPositions added in v1.26.2 

func (db *DB) ListStreamPositions(ctx context.Context, opts ListStreamPositions) (result ListStreamPositionsResult, err error)

ListStreamPositions lists specified stream segment positions.

func (*DB) MigrateToLatest added in v1.26.2 

func (db *DB) MigrateToLatest(ctx context.Context) error

MigrateToLatest migrates database to the latest version.

func (*DB) Ping added in v1.26.2 

func (db *DB) Ping(ctx context.Context) error

Ping checks whether connection has been established.

func (*DB) PostgresMigration added in v1.26.2 

func (db *DB) PostgresMigration() *migrate.Migration

PostgresMigration returns steps needed for migrating postgres database.

func (*DB) TestingAllCommittedObjects added in v1.26.2 

func (db *DB) TestingAllCommittedObjects(ctx context.Context, projectID uuid.UUID, bucketName string) (objects []ObjectEntry, err error)

TestingAllCommittedObjects gets all objects from bucket. Use only for testing purposes.

func (*DB) TestingAllObjectSegments added in v1.26.2 

func (db *DB) TestingAllObjectSegments(ctx context.Context, objectLocation ObjectLocation) (segments []Segment, err error)

TestingAllObjectSegments gets all segments for given object. Use only for testing purposes.

func (*DB) TestingAllObjects added in v1.26.2 

func (db *DB) TestingAllObjects(ctx context.Context) (objects []Object, err error)

TestingAllObjects gets all objects. Use only for testing purposes.

func (*DB) TestingAllPendingObjects added in v1.26.2 

func (db *DB) TestingAllPendingObjects(ctx context.Context, projectID uuid.UUID, bucketName string) (objects []ObjectEntry, err error)

TestingAllPendingObjects gets all objects from bucket. Use only for testing purposes.

func (*DB) TestingAllSegments added in v1.26.2 

func (db *DB) TestingAllSegments(ctx context.Context) (segments []Segment, err error)

TestingAllSegments gets all segments. Use only for testing purposes.

func (*DB) TestingDeleteAll added in v1.26.2 

func (db *DB) TestingDeleteAll(ctx context.Context) (err error)

TestingDeleteAll deletes all objects and segments from the database.

func (*DB) TestingGetState added in v1.26.2 

func (db *DB) TestingGetState(ctx context.Context) (_ *RawState, err error)

TestingGetState returns the state of the database.

func (*DB) UnderlyingTagSQL added in v1.26.2 

func (db *DB) UnderlyingTagSQL() tagsql.DB

UnderlyingTagSQL returns *tagsql.DB. TODO: remove.

func (*DB) UpdateObjectMetadata added in v1.26.2 

func (db *DB) UpdateObjectMetadata(ctx context.Context, opts UpdateObjectMetadata) (err error)

UpdateObjectMetadata updates an object metadata.

func (*DB) UpdateSegmentPieces added in v1.26.2 

func (db *DB) UpdateSegmentPieces(ctx context.Context, opts UpdateSegmentPieces) (err error)

UpdateSegmentPieces updates pieces for specified segment. If provided old pieces won't match current database state update will fail.

type DeleteBucketObjects added in v1.26.2 

type DeleteBucketObjects struct {
	Bucket    BucketLocation
	BatchSize int

	// DeletePieces is called for every batch of objects.
	// Slice `segments` will be reused between calls.
	DeletePieces func(ctx context.Context, segments []DeletedSegmentInfo) error
}

DeleteBucketObjects contains arguments for deleting a whole bucket.

type DeleteExpiredObjects added in v1.26.2 

type DeleteExpiredObjects struct {
	ExpiredBefore  time.Time
	AsOfSystemTime time.Time
	BatchSize      int
}

DeleteExpiredObjects contains all the information necessary to delete expired objects and segments.

type DeleteObjectAnyStatusAllVersions added in v1.26.2 

type DeleteObjectAnyStatusAllVersions struct {
	ObjectLocation
}

DeleteObjectAnyStatusAllVersions contains arguments necessary for deleting all object versions.

type DeleteObjectExactVersion added in v1.26.2 

type DeleteObjectExactVersion struct {
	Version Version
	ObjectLocation
}

DeleteObjectExactVersion contains arguments necessary for deleting an exact version of object.

func (*DeleteObjectExactVersion) Verify added in v1.26.2 

func (obj *DeleteObjectExactVersion) Verify() error

Verify delete object fields.

type DeleteObjectLatestVersion added in v1.26.2 

type DeleteObjectLatestVersion struct {
	ObjectLocation
}

DeleteObjectLatestVersion contains arguments necessary for deleting latest object version.

type DeleteObjectResult added in v1.26.2 

type DeleteObjectResult struct {
	Objects  []Object
	Segments []DeletedSegmentInfo
}

DeleteObjectResult result of deleting object.

type DeleteObjectsAllVersions added in v1.26.2 

type DeleteObjectsAllVersions struct {
	Locations []ObjectLocation
}

DeleteObjectsAllVersions contains arguments necessary for deleting all versions of multiple objects from the same bucket.

func (*DeleteObjectsAllVersions) Verify added in v1.26.2 

func (delete *DeleteObjectsAllVersions) Verify() error

Verify delete objects fields.

type DeletePendingObject added in v1.26.2 

type DeletePendingObject struct {
	ObjectLocation
	Version
	StreamID uuid.UUID
}

DeletePendingObject contains arguments necessary for deleting a pending object.

func (*DeletePendingObject) Verify added in v1.26.2 

func (opts *DeletePendingObject) Verify() error

Verify verifies delete pending object fields validity.

type DeletedSegmentInfo added in v1.26.2 

type DeletedSegmentInfo struct {
	RootPieceID storj.PieceID
	Pieces      Pieces
}

DeletedSegmentInfo info about deleted segment.

type EnsureNodeAliases added in v1.26.2 

type EnsureNodeAliases struct {
	Nodes []storj.NodeID
}

EnsureNodeAliases contains arguments necessary for creating NodeAlias-es.

type GetLatestObjectLastSegment added in v1.26.2 

type GetLatestObjectLastSegment struct {
	ObjectLocation
}

GetLatestObjectLastSegment contains arguments necessary for fetching a last segment information.

type GetObjectExactVersion added in v1.26.2 

type GetObjectExactVersion struct {
	Version Version
	ObjectLocation
}

GetObjectExactVersion contains arguments necessary for fetching an information about exact object version.

func (*GetObjectExactVersion) Verify added in v1.26.2 

func (obj *GetObjectExactVersion) Verify() error

Verify verifies get object reqest fields.

type GetObjectLatestVersion added in v1.26.2 

type GetObjectLatestVersion struct {
	ObjectLocation
}

GetObjectLatestVersion contains arguments necessary for fetching an object information for latest version.

type GetSegmentByLocation added in v1.26.2 

type GetSegmentByLocation struct {
	SegmentLocation
}

GetSegmentByLocation contains arguments necessary for fetching a segment on specific segment location.

type GetSegmentByOffset added in v1.26.2 

type GetSegmentByOffset struct {
	ObjectLocation
	PlainOffset int64
}

GetSegmentByOffset contains arguments necessary for fetching a segment information.

type GetSegmentByPosition added in v1.26.2 

type GetSegmentByPosition struct {
	StreamID uuid.UUID
	Position SegmentPosition
}

GetSegmentByPosition contains arguments necessary for fetching a segment on specific position.

func (*GetSegmentByPosition) Verify added in v1.26.2 

func (seg *GetSegmentByPosition) Verify() error

Verify verifies get segment request fields.

type GetStreamPieceCountByNodeID added in v1.26.2 

type GetStreamPieceCountByNodeID struct {
	StreamID uuid.UUID
}

GetStreamPieceCountByNodeID contains arguments for GetStreamPieceCountByNodeID.

type IterateCursor added in v1.26.2 

type IterateCursor struct {
	Key     ObjectKey
	Version Version
}

IterateCursor is a cursor used during iteration through objects.

type IterateLoopObjects added in v1.26.2 

type IterateLoopObjects struct {
	BatchSize int

	AsOfSystemTime time.Time
}

IterateLoopObjects contains arguments necessary for listing objects in metabase.

func (*IterateLoopObjects) Verify added in v1.26.2 

func (opts *IterateLoopObjects) Verify() error

Verify verifies get object request fields.

type IterateLoopStreams added in v1.26.2 

type IterateLoopStreams struct {
	StreamIDs []uuid.UUID

	AsOfSystemTime time.Time
}

IterateLoopStreams contains arguments necessary for listing multiple streams segments.

type IterateObjects added in v1.26.2 

type IterateObjects struct {
	ProjectID  uuid.UUID
	BucketName string
	BatchSize  int
	Prefix     ObjectKey
	Cursor     IterateCursor
}

IterateObjects contains arguments necessary for listing objects in a bucket.

func (*IterateObjects) Verify added in v1.26.2 

func (opts *IterateObjects) Verify() error

Verify verifies get object request fields.

type IterateObjectsWithStatus added in v1.26.2 

type IterateObjectsWithStatus struct {
	ProjectID  uuid.UUID
	BucketName string
	Recursive  bool
	BatchSize  int
	Prefix     ObjectKey
	Cursor     IterateCursor
	Status     ObjectStatus
}

IterateObjectsWithStatus contains arguments necessary for listing objects in a bucket.

func (*IterateObjectsWithStatus) Verify added in v1.26.2 

func (opts *IterateObjectsWithStatus) Verify() error

Verify verifies get object request fields.

type IteratePendingObjectsByKey added in v1.26.2 

type IteratePendingObjectsByKey struct {
	ObjectLocation
	BatchSize int
	Cursor    StreamIDCursor
}

IteratePendingObjectsByKey contains arguments necessary for listing pending objects by ObjectKey.

func (*IteratePendingObjectsByKey) Verify added in v1.26.2 

func (opts *IteratePendingObjectsByKey) Verify() error

Verify verifies get object request fields.

type ListSegments added in v1.26.2 

type ListSegments struct {
	StreamID uuid.UUID
	Cursor   SegmentPosition
	Limit    int
}

ListSegments contains arguments necessary for listing stream segments.

type ListSegmentsResult added in v1.26.2 

type ListSegmentsResult struct {
	Segments []Segment
	More     bool
}

ListSegmentsResult result of listing segments.

type ListStreamPositions added in v1.26.2 

type ListStreamPositions struct {
	StreamID uuid.UUID
	Cursor   SegmentPosition
	Limit    int

	Range *StreamRange
}

ListStreamPositions contains arguments necessary for listing stream segments.

type ListStreamPositionsResult added in v1.26.2 

type ListStreamPositionsResult struct {
	Segments []SegmentPositionInfo
	More     bool
}

ListStreamPositionsResult result of listing segments.

type LoopObjectEntry added in v1.26.2 

type LoopObjectEntry struct {
	ObjectStream                     // metrics, repair, tally
	ExpiresAt             *time.Time // tally
	SegmentCount          int32      // metrics
	EncryptedMetadataSize int        // tally
}

LoopObjectEntry contains information about object needed by metainfo loop.

type LoopObjectsIterator added in v1.26.2 

type LoopObjectsIterator interface {
	Next(ctx context.Context, item *LoopObjectEntry) bool
}

LoopObjectsIterator iterates over a sequence of LoopObjectEntry items.

type LoopSegmentEntry added in v1.26.2 

type LoopSegmentEntry struct {
	StreamID      uuid.UUID
	Position      SegmentPosition
	CreatedAt     *time.Time // repair
	RepairedAt    *time.Time // repair
	RootPieceID   storj.PieceID
	EncryptedSize int32 // size of the whole segment (not a piece)
	Redundancy    storj.RedundancyScheme
	Pieces        Pieces
}

LoopSegmentEntry contains information about segment metadata needed by metainfo loop.

func (LoopSegmentEntry) Inline added in v1.26.2 

func (s LoopSegmentEntry) Inline() bool

Inline returns true if segment is inline.

type NodeAlias added in v1.26.2 

type NodeAlias int32

NodeAlias is a metabase local alias for NodeID-s to reduce segment table size.

type NodeAliasCache added in v1.26.2 

type NodeAliasCache struct {
	// contains filtered or unexported fields
}

NodeAliasCache is a write-through cache for looking up node ID and alias mapping.

func NewNodeAliasCache added in v1.26.2 

func NewNodeAliasCache(db NodeAliasDB) *NodeAliasCache

NewNodeAliasCache creates a new cache using the specified database.

func (*NodeAliasCache) Aliases added in v1.26.2 

func (cache *NodeAliasCache) Aliases(ctx context.Context, nodes []storj.NodeID) ([]NodeAlias, error)

Aliases returns node aliases corresponding to the node ID-s, adding missing node ID-s to the database when needed.

func (*NodeAliasCache) ConvertAliasesToPieces added in v1.26.2 

func (cache *NodeAliasCache) ConvertAliasesToPieces(ctx context.Context, aliasPieces AliasPieces) (_ Pieces, err error)

ConvertAliasesToPieces converts alias pieces to pieces.

func (*NodeAliasCache) ConvertPiecesToAliases added in v1.26.2 

func (cache *NodeAliasCache) ConvertPiecesToAliases(ctx context.Context, pieces Pieces) (_ AliasPieces, err error)

ConvertPiecesToAliases converts pieces to alias pieces.

func (*NodeAliasCache) Nodes added in v1.26.2 

func (cache *NodeAliasCache) Nodes(ctx context.Context, aliases []NodeAlias) ([]storj.NodeID, error)

Nodes returns node ID-s corresponding to the aliases, refreshing the cache once when an alias is missing. This results in an error when the alias is not in the database.

type NodeAliasDB added in v1.26.2 

type NodeAliasDB interface {
	EnsureNodeAliases(ctx context.Context, opts EnsureNodeAliases) error
	ListNodeAliases(ctx context.Context) (_ []NodeAliasEntry, err error)
}

NodeAliasDB is an interface for looking up node alises.

type NodeAliasEntry added in v1.26.2 

type NodeAliasEntry struct {
	ID    storj.NodeID
	Alias NodeAlias
}

NodeAliasEntry is a mapping between NodeID and NodeAlias.

type NodeAliasMap added in v1.26.2 

type NodeAliasMap struct {
	// contains filtered or unexported fields
}

NodeAliasMap contains bidirectional mapping between node ID and a NodeAlias.

func NewNodeAliasMap added in v1.26.2 

func NewNodeAliasMap(entries []NodeAliasEntry) *NodeAliasMap

NewNodeAliasMap creates a new alias map from the given entries.

func (*NodeAliasMap) Aliases added in v1.26.2 

func (m *NodeAliasMap) Aliases(nodes []storj.NodeID) (xs []NodeAlias, missing []storj.NodeID)

Aliases returns alises-s for the given node ID-s and node ID-s that are not in this map.

func (*NodeAliasMap) ContainsAll added in v1.26.2 

func (m *NodeAliasMap) ContainsAll(nodeIDs []storj.NodeID, nodeAliases []NodeAlias) bool

ContainsAll returns true when the table contains all entries.

func (*NodeAliasMap) Merge added in v1.26.2 

func (m *NodeAliasMap) Merge(other *NodeAliasMap)

Merge merges the other map into m.

func (*NodeAliasMap) Node added in v1.26.2 

func (m *NodeAliasMap) Node(alias NodeAlias) (x storj.NodeID, ok bool)

Node returns NodeID for the given alias.

func (*NodeAliasMap) Nodes added in v1.26.2 

func (m *NodeAliasMap) Nodes(aliases []NodeAlias) (xs []storj.NodeID, missing []NodeAlias)

Nodes returns NodeID-s for the given aliases and aliases that are not in this map.

func (*NodeAliasMap) Size added in v1.26.2 

func (m *NodeAliasMap) Size() int

Size returns the number of entries in this map.

type Object added in v1.26.2 

type Object RawObject

Object object metadata. TODO define separated struct.

func (*Object) IsMigrated added in v1.27.0 

func (obj *Object) IsMigrated() bool

IsMigrated returns whether the object comes from PointerDB. Pointer objects are special that they are missing some information.

  • TotalPlainSize = 0 and FixedSegmentSize = 0.
  • Segment.PlainOffset = 0, Segment.PlainSize = 0

type ObjectEntry added in v1.22.1 

type ObjectEntry struct {
	IsPrefix bool

	ObjectKey ObjectKey
	Version   Version
	StreamID  uuid.UUID

	CreatedAt time.Time
	ExpiresAt *time.Time

	Status       ObjectStatus
	SegmentCount int32

	EncryptedMetadataNonce        []byte
	EncryptedMetadata             []byte
	EncryptedMetadataEncryptedKey []byte

	TotalPlainSize     int64
	TotalEncryptedSize int64
	FixedSegmentSize   int32

	Encryption storj.EncryptionParameters

	// ZombieDeletionDeadline defines when the pending raw object should be deleted from the database.
	// This is as a safeguard against objects that failed to upload and the client has not indicated
	// whether they want to continue uploading or delete the already uploaded data.
	ZombieDeletionDeadline *time.Time
}

ObjectEntry contains information about an item in a bucket.

type ObjectKey 

type ObjectKey string

ObjectKey is an encrypted object key encoded using Path Component Encoding. It is not ascii safe.

type ObjectLocation 

type ObjectLocation struct {
	ProjectID  uuid.UUID
	BucketName string
	ObjectKey  ObjectKey
}

ObjectLocation is decoded object key information.

func (ObjectLocation) Bucket 

func (obj ObjectLocation) Bucket() BucketLocation

Bucket returns bucket location this object belongs to.

func (ObjectLocation) Verify added in v1.26.2 

func (obj ObjectLocation) Verify() error

Verify object location fields.

type ObjectStatus added in v1.26.2 

type ObjectStatus byte

ObjectStatus defines the statuses that the object might be in.

type ObjectStream added in v1.26.2 

type ObjectStream struct {
	ProjectID  uuid.UUID
	BucketName string
	ObjectKey  ObjectKey
	Version    Version
	StreamID   uuid.UUID
}

ObjectStream uniquely defines an object and stream.

TODO: figure out whether ther's a better name.

func (*ObjectStream) Location added in v1.26.2 

func (obj *ObjectStream) Location() ObjectLocation

Location returns object location.

func (*ObjectStream) Verify added in v1.26.2 

func (obj *ObjectStream) Verify() error

Verify object stream fields.

type ObjectsIterator added in v1.26.2 

type ObjectsIterator interface {
	Next(ctx context.Context, item *ObjectEntry) bool
}

ObjectsIterator iterates over a sequence of ObjectEntry items.

type Piece added in v1.16.1 

type Piece struct {
	Number      uint16
	StorageNode storj.NodeID
}

Piece defines information for a segment piece.

type Pieces added in v1.16.1 

type Pieces []Piece

Pieces defines information for pieces.

func (Pieces) Equal added in v1.26.2 

func (p Pieces) Equal(pieces Pieces) bool

Equal checks if Pieces structures are equal.

func (Pieces) Len added in v1.26.2 

func (p Pieces) Len() int

Len is the number of pieces.

func (Pieces) Less added in v1.26.2 

func (p Pieces) Less(i, j int) bool

Less reports whether the piece with index i should sort before the piece with index j.

func (*Pieces) Scan added in v1.26.2 

func (pieces *Pieces) Scan(value interface{}) error

Scan implements sql.Scanner interface.

func (Pieces) Swap added in v1.26.2 

func (p Pieces) Swap(i, j int)

Swap swaps the pieces with indexes i and j.

func (Pieces) Value added in v1.26.2 

func (pieces Pieces) Value() (driver.Value, error)

Value implements sql/driver.Valuer interface.

func (Pieces) Verify added in v1.26.2 

func (p Pieces) Verify() error

Verify verifies pieces.

type RawObject added in v1.26.2 

type RawObject struct {
	ObjectStream

	CreatedAt time.Time
	ExpiresAt *time.Time

	Status       ObjectStatus
	SegmentCount int32

	EncryptedMetadataNonce        []byte
	EncryptedMetadata             []byte
	EncryptedMetadataEncryptedKey []byte

	// TotalPlainSize is 0 for a migrated object.
	TotalPlainSize     int64
	TotalEncryptedSize int64
	// FixedSegmentSize is 0 for a migrated object.
	FixedSegmentSize int32

	Encryption storj.EncryptionParameters

	// ZombieDeletionDeadline defines when the pending raw object should be deleted from the database.
	// This is as a safeguard against objects that failed to upload and the client has not indicated
	// whether they want to continue uploading or delete the already uploaded data.
	ZombieDeletionDeadline *time.Time
}

RawObject defines the full object that is stored in the database. It should be rarely used directly.

type RawSegment added in v1.26.2 

type RawSegment struct {
	StreamID uuid.UUID
	Position SegmentPosition

	CreatedAt  *time.Time // TODO: make it non-nilable after we migrate all existing segments to have creation time
	RepairedAt *time.Time

	RootPieceID       storj.PieceID
	EncryptedKeyNonce []byte
	EncryptedKey      []byte

	EncryptedSize int32 // size of the whole segment (not a piece)
	// PlainSize is 0 for a migrated object.
	PlainSize int32
	// PlainOffset is 0 for a migrated object.
	PlainOffset   int64
	EncryptedETag []byte

	Redundancy storj.RedundancyScheme

	InlineData []byte
	Pieces     Pieces
}

RawSegment defines the full segment that is stored in the database. It should be rarely used directly.

type RawState added in v1.26.2 

type RawState struct {
	Objects  []RawObject
	Segments []RawSegment
}

RawState contains full state of a table.

type Segment added in v1.22.1 

type Segment RawSegment

Segment segment metadata. TODO define separated struct.

func (Segment) Inline added in v1.26.2 

func (s Segment) Inline() bool

Inline returns true if segment is inline.

type SegmentIterator added in v1.26.2 

type SegmentIterator func(segment *LoopSegmentEntry) bool

SegmentIterator returns the next segment.

type SegmentKey 

type SegmentKey []byte

SegmentKey is an encoded metainfo key. This is used as the key in pointerdb key-value store.

type SegmentLocation 

type SegmentLocation struct {
	ProjectID  uuid.UUID
	BucketName string
	ObjectKey  ObjectKey
	Position   SegmentPosition
}

SegmentLocation is decoded segment key information.

func ParseSegmentKey 

func ParseSegmentKey(encoded SegmentKey) (SegmentLocation, error)

ParseSegmentKey parses an segment key into segment location.

func (SegmentLocation) Bucket 

func (seg SegmentLocation) Bucket() BucketLocation

Bucket returns bucket location this segment belongs to.

func (SegmentLocation) Encode 

func (seg SegmentLocation) Encode() SegmentKey

Encode converts segment location into a segment key.

func (SegmentLocation) Object 

func (seg SegmentLocation) Object() ObjectLocation

Object returns the object location associated with this segment location.

func (SegmentLocation) Verify added in v1.26.2 

func (seg SegmentLocation) Verify() error

Verify segment location fields.

type SegmentPosition added in v1.22.1 

type SegmentPosition struct {
	Part  uint32
	Index uint32
}

SegmentPosition is segment part and index combined.

func SegmentPositionFromEncoded added in v1.26.2 

func SegmentPositionFromEncoded(v uint64) SegmentPosition

SegmentPositionFromEncoded decodes an uint64 into a SegmentPosition.

func (SegmentPosition) Encode added in v1.26.2 

func (pos SegmentPosition) Encode() uint64

Encode encodes a segment position into an uint64, that can be stored in a database.

func (SegmentPosition) Less added in v1.26.2 

func (pos SegmentPosition) Less(b SegmentPosition) bool

Less returns whether pos should before b.

func (*SegmentPosition) Scan added in v1.26.2 

func (params *SegmentPosition) Scan(value interface{}) error

Scan implements sql.Scanner interface.

func (SegmentPosition) Value added in v1.26.2 

func (params SegmentPosition) Value() (driver.Value, error)

Value implements sql/driver.Valuer interface.

type SegmentPositionInfo added in v1.26.2 

type SegmentPositionInfo struct {
	Position SegmentPosition
	// PlainSize is 0 for a migrated object.
	PlainSize int32
	// PlainOffset is 0 for a migrated object.
	PlainOffset       int64
	CreatedAt         *time.Time // TODO: make it non-nilable after we migrate all existing segments to have creation time
	EncryptedETag     []byte
	EncryptedKeyNonce []byte
	EncryptedKey      []byte
}

SegmentPositionInfo contains information for segment position.

type StreamIDCursor added in v1.26.2 

type StreamIDCursor struct {
	StreamID uuid.UUID
}

StreamIDCursor is a cursor used during iteration through streamIDs of a pending object.

type StreamRange added in v1.27.0 

type StreamRange struct {
	PlainStart int64
	PlainLimit int64 // limit is exclusive
}

StreamRange allows to limit stream positions based on the plain offsets.

type UpdateObjectMetadata added in v1.26.2 

type UpdateObjectMetadata struct {
	ObjectStream

	EncryptedMetadata             []byte
	EncryptedMetadataNonce        []byte
	EncryptedMetadataEncryptedKey []byte
}

UpdateObjectMetadata contains arguments necessary for updating an object metadata.

type UpdateSegmentPieces added in v1.26.2 

type UpdateSegmentPieces struct {
	StreamID uuid.UUID
	Position SegmentPosition

	OldPieces Pieces

	NewRedundancy storj.RedundancyScheme
	NewPieces     Pieces

	NewRepairedAt time.Time // sets new time of last segment repair (optional).
}

UpdateSegmentPieces contains arguments necessary for updating segment pieces.

type Version added in v1.26.2 

type Version int64

Version is used to uniquely identify objects with the same key.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.