Documentation ¶
Index ¶
- Constants
- Variables
- func New(log *zap.Logger, dir *Dir, config Config) blobstore.Blobs
- func NewAt(log *zap.Logger, path string, config Config) (blobstore.Blobs, error)
- type Config
- type CorruptDataError
- type Dir
- func (dir *Dir) Commit(ctx context.Context, file *os.File, ref blobstore.BlobRef, ...) (err error)
- func (dir *Dir) CreateTemporaryFile(ctx context.Context, prealloc int64) (_ *os.File, err error)
- func (dir *Dir) CreateVerificationFile(ctx context.Context, id storj.NodeID) error
- func (dir *Dir) Delete(ctx context.Context, ref blobstore.BlobRef) (err error)
- func (dir *Dir) DeleteNamespace(ctx context.Context, ref []byte) (err error)
- func (dir *Dir) DeleteTemporary(ctx context.Context, file *os.File) (err error)
- func (dir *Dir) DeleteTrashNamespace(ctx context.Context, namespace []byte) (err error)
- func (dir *Dir) DeleteWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (err error)
- func (dir *Dir) EmptyTrash(ctx context.Context, namespace []byte, trashedBefore time.Time) (bytesEmptied int64, deletedKeys [][]byte, err error)
- func (dir *Dir) GarbageCollect(ctx context.Context) (err error)
- func (dir *Dir) Info(ctx context.Context) (DiskInfo, error)
- func (dir *Dir) ListNamespaces(ctx context.Context) (ids [][]byte, err error)
- func (dir *Dir) Open(ctx context.Context, ref blobstore.BlobRef) (_ *os.File, _ blobstore.FormatVersion, err error)
- func (dir *Dir) OpenWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (_ *os.File, err error)
- func (dir *Dir) Path() string
- func (dir *Dir) ReplaceTrashnow(trashnow func() time.Time)
- func (dir *Dir) RestoreTrash(ctx context.Context, namespace []byte) (keysRestored [][]byte, err error)
- func (dir *Dir) Stat(ctx context.Context, ref blobstore.BlobRef) (_ blobstore.BlobInfo, err error)
- func (dir *Dir) StatWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (_ blobstore.BlobInfo, err error)
- func (dir *Dir) Trash(ctx context.Context, ref blobstore.BlobRef) (err error)
- func (dir *Dir) TrashWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (err error)
- func (dir *Dir) TryRestoreTrashPiece(ctx context.Context, ref blobstore.BlobRef) (err error)
- func (dir *Dir) Verify(ctx context.Context, id storj.NodeID) error
- func (dir *Dir) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(blobstore.BlobInfo) error) (err error)
- type DiskInfo
Constants ¶
const ( // FormatV0 is the identifier for storage format v0, which also corresponds to an absence of // format version information. FormatV0 blobstore.FormatVersion = 0 // FormatV1 is the identifier for storage format v1. FormatV1 blobstore.FormatVersion = 1 )
const ( // MaxFormatVersionSupported is the highest supported storage format version for reading, and // the only supported storage format version for writing. If stored blobs claim a higher // storage format version than this, or a caller requests _writing_ a storage format version // which is not this, this software will not know how to perform the read or write and an error // will be returned. MaxFormatVersionSupported = FormatV1 // MinFormatVersionSupported is the lowest supported storage format version for reading. If // stored blobs claim a lower storage format version than this, this software will not know how // to perform the read and an error will be returned. MinFormatVersionSupported = FormatV0 )
Variables ¶
var ( // Error is the default filestore error class. Error = errs.Class("filestore error") // ErrIsDir is the error returned when we encounter a directory named like a blob file // while traversing a blob namespace. ErrIsDir = Error.New("file is a directory") )
var DefaultConfig = Config{ WriteBufferSize: 128 * memory.KiB, }
DefaultConfig is the default value for Config.
Functions ¶
Types ¶
type Config ¶
type Config struct {
WriteBufferSize memory.Size `help:"in-memory buffer for uploads" default:"128KiB"`
}
Config is configuration for the blob store.
type CorruptDataError ¶
type CorruptDataError struct {
// contains filtered or unexported fields
}
CorruptDataError represents a filesystem or disk error which indicates data corruption.
We use a custom error type here so that we can add explanatory information and wrap the original error at the same time.
func (CorruptDataError) Error ¶
func (cde CorruptDataError) Error() string
Error returns an error string describing the condition.
func (CorruptDataError) Path ¶
func (cde CorruptDataError) Path() string
Path returns the path at which the error was encountered.
func (CorruptDataError) Unwrap ¶
func (cde CorruptDataError) Unwrap() error
Unwrap unwraps the error.
type Dir ¶
type Dir struct {
// contains filtered or unexported fields
}
Dir represents single folder for storing blobs.
func (*Dir) Commit ¶
func (dir *Dir) Commit(ctx context.Context, file *os.File, ref blobstore.BlobRef, formatVersion blobstore.FormatVersion) (err error)
Commit commits the temporary file to permanent storage.
func (*Dir) CreateTemporaryFile ¶
CreateTemporaryFile creates a preallocated temporary file in the temp directory prealloc preallocates file to make writing faster.
func (*Dir) CreateVerificationFile ¶
CreateVerificationFile creates a file to be used for storage directory verification.
func (*Dir) Delete ¶
Delete deletes blobs with the specified ref (in all supported storage formats).
It doesn't return an error if the blob is not found for any reason or it cannot be deleted at this moment and it's delayed.
func (*Dir) DeleteNamespace ¶
DeleteNamespace deletes blobs folder for a specific namespace.
func (*Dir) DeleteTemporary ¶
DeleteTemporary deletes a temporary file.
func (*Dir) DeleteTrashNamespace ¶ added in v1.88.2
DeleteTrashNamespace deletes the entire trash namespace.
func (*Dir) DeleteWithStorageFormat ¶
func (dir *Dir) DeleteWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (err error)
DeleteWithStorageFormat deletes the blob with the specified ref for one specific format version. The method tries the following strategies, in order of preference until one succeeds:
* moves the blob to garbage dir. * directly deletes the blob. * push the blobs to queue for retrying later.
It doesn't return an error if the piece isn't found for any reason.
func (*Dir) EmptyTrash ¶
func (dir *Dir) EmptyTrash(ctx context.Context, namespace []byte, trashedBefore time.Time) (bytesEmptied int64, deletedKeys [][]byte, err error)
EmptyTrash walks the trash files for the given namespace and deletes any file whose mtime is older than trashedBefore. The mtime is modified when Trash is called.
func (*Dir) GarbageCollect ¶
GarbageCollect collects files that are pending deletion.
func (*Dir) ListNamespaces ¶
ListNamespaces finds all known namespace IDs in use in local storage. They are not guaranteed to contain any blobs.
func (*Dir) Open ¶
func (dir *Dir) Open(ctx context.Context, ref blobstore.BlobRef) (_ *os.File, _ blobstore.FormatVersion, err error)
Open opens the file with the specified ref. It may need to check in more than one location in order to find the blob, if it was stored with an older version of the storage node software. In cases where the storage format version of a blob is already known, OpenWithStorageFormat() will generally be a better choice.
func (*Dir) OpenWithStorageFormat ¶
func (dir *Dir) OpenWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (_ *os.File, err error)
OpenWithStorageFormat opens an already-located blob file with a known storage format version, which avoids the potential need to search through multiple storage formats to find the blob.
func (*Dir) ReplaceTrashnow ¶
ReplaceTrashnow is a helper for tests to replace the trashnow function used when moving files to the trash.
func (*Dir) RestoreTrash ¶
func (dir *Dir) RestoreTrash(ctx context.Context, namespace []byte) (keysRestored [][]byte, err error)
RestoreTrash moves every piece in the trash folder back into blobsdir.
func (*Dir) Stat ¶
Stat looks up disk metadata on the blob file. It may need to check in more than one location in order to find the blob, if it was stored with an older version of the storage node software. In cases where the storage format version of a blob is already known, StatWithStorageFormat() will generally be a better choice.
func (*Dir) StatWithStorageFormat ¶
func (dir *Dir) StatWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (_ blobstore.BlobInfo, err error)
StatWithStorageFormat looks up disk metadata on the blob file with the given storage format version. This avoids the need for checking for the file in multiple different storage format types.
func (*Dir) Trash ¶
Trash moves the piece specified by ref to the trashdir for every format version.
func (*Dir) TrashWithStorageFormat ¶
func (dir *Dir) TrashWithStorageFormat(ctx context.Context, ref blobstore.BlobRef, formatVer blobstore.FormatVersion) (err error)
TrashWithStorageFormat moves the piece specified by ref to the trashdir for the specified format version.
func (*Dir) TryRestoreTrashPiece ¶ added in v1.88.2
TryRestoreTrashPiece attempts to restore a piece from the trash if it exists. It returns nil if the piece was restored, or an error if the piece was not in the trash or could not be restored.
func (*Dir) Verify ¶
Verify verifies that the storage directory is correct by checking for the existence and validity of the verification file.
func (*Dir) WalkNamespace ¶
func (dir *Dir) WalkNamespace(ctx context.Context, namespace []byte, walkFunc func(blobstore.BlobInfo) error) (err error)
WalkNamespace executes walkFunc for each locally stored blob, stored with storage format V1 or greater, in the given namespace. If walkFunc returns a non-nil error, WalkNamespace will stop iterating and return the error immediately. The ctx parameter is intended specifically to allow canceling iteration early.