lazyfilewalker

Documentation

Index

• Constants
• type Config
  • func (config *Config) Args() []string
• type GCFilewalkerRequest
• type GCFilewalkerResponse
• type Supervisor
  • func NewSupervisor(log *zap.Logger, config Config, executable string) *Supervisor
  • func (fw *Supervisor) TestingSetGCCmd(cmd execwrapper.Command)
  • func (fw *Supervisor) TestingSetTrashCleanupCmd(cmd execwrapper.Command)
  • func (fw *Supervisor) TestingSetUsedSpaceCmd(cmd execwrapper.Command)
  • func (fw *Supervisor) WalkAndComputeSpaceUsedBySatellite(ctx context.Context, satelliteID storj.NodeID) (piecesTotal int64, piecesContentSize int64, pieceCount int64, err error)
  • func (fw *Supervisor) WalkCleanupTrash(ctx context.Context, satelliteID storj.NodeID, dateBefore time.Time) (bytesDeleted int64, keysDeleted []storj.PieceID, err error)
  • func (fw *Supervisor) WalkSatellitePiecesToTrash(ctx context.Context, satelliteID storj.NodeID, createdBefore time.Time, ...) (piecesCount, piecesSkipped int64, err error)
• type TrashCleanupRequest
• type TrashCleanupResponse
• type TrashHandler
  • func NewTrashHandler(log *zap.Logger, trashFunc func(pieceID storj.PieceID) error) *TrashHandler
  • func (t *TrashHandler) Decode(v interface{}) error
  • func (t *TrashHandler) Write(b []byte) (n int, err error)
• type UsedSpaceRequest
• type UsedSpaceResponse

Constants

const (
	// UsedSpaceFilewalkerCmdName is the name of the used-space-filewalker subcommand.
	UsedSpaceFilewalkerCmdName = "used-space-filewalker"
	// GCFilewalkerCmdName is the name of the gc-filewalker subcommand.
	GCFilewalkerCmdName = "gc-filewalker"
	// TrashCleanupFilewalkerCmdName is the name of the trash-cleanup-filewalker subcommand.
	TrashCleanupFilewalkerCmdName = "trash-cleanup-filewalker"
)

Types

type Config

type Config struct {
	// TODO: just trying to match the names in storagenodedb.Config. Change these to be more descriptive.
	Storage   string `help:"path to the storage database directory"`
	Info      string `help:"path to the piecestore db"`
	Info2     string `help:"path to the info database"`
	Driver    string `help:"database driver to use" default:"sqlite3"`
	Pieces    string `help:"path to store pieces in"`
	Cache     string `` /* 131-byte string literal not displayed */
	Filestore filestore.Config

	LowerIOPriority bool `help:"if true, the process will run with lower IO priority" default:"true"`
}

Config is the config for lazyfilewalker process.

func (*Config) Args

func (config *Config) Args() []string

Args returns the flags to be passed lazyfilewalker process.

type GCFilewalkerRequest

type GCFilewalkerRequest struct {
	SatelliteID   storj.NodeID `json:"satelliteID"`
	BloomFilter   []byte       `json:"bloomFilter"`
	CreatedBefore time.Time    `json:"createdBefore"`
}

GCFilewalkerRequest is the request struct for the gc-filewalker process.

type GCFilewalkerResponse

type GCFilewalkerResponse struct {
	// PieceIDs is the list of trash pieces that were found.
	// Final message will not return any pieceIDs.
	PieceIDs           []storj.PieceID `json:"pieceIDs"`
	PiecesSkippedCount int64           `json:"piecesSkippedCount"`
	PiecesCount        int64           `json:"piecesCount"`
	// Completed indicates if this is the final message.
	Completed bool `json:"completed"`
}

GCFilewalkerResponse is the response struct for the gc-filewalker process.

type Supervisor

type Supervisor struct {
	// contains filtered or unexported fields
}

Supervisor manages the lazyfilewalker subprocesses.

TODO: we should keep track of the number of subprocesses we have running and limit it to a configurable number, and queue them, since they are run per satellite.

func NewSupervisor

func NewSupervisor(log *zap.Logger, config Config, executable string) *Supervisor

NewSupervisor creates a new lazy filewalker Supervisor.

func (*Supervisor) TestingSetGCCmd

func (fw *Supervisor) TestingSetGCCmd(cmd execwrapper.Command)

TestingSetGCCmd sets the command for the gc-filewalker subprocess. The cmd acts as a replacement for the subprocess.

func (*Supervisor) TestingSetTrashCleanupCmd

func (fw *Supervisor) TestingSetTrashCleanupCmd(cmd execwrapper.Command)

TestingSetTrashCleanupCmd sets the command for the trash cleanup filewalker subprocess. The cmd acts as a replacement for the subprocess.

func (*Supervisor) TestingSetUsedSpaceCmd

func (fw *Supervisor) TestingSetUsedSpaceCmd(cmd execwrapper.Command)

TestingSetUsedSpaceCmd sets the command for the used-space-filewalker subprocess. The cmd acts as a replacement for the subprocess.

func (*Supervisor) WalkAndComputeSpaceUsedBySatellite

func (fw *Supervisor) WalkAndComputeSpaceUsedBySatellite(ctx context.Context, satelliteID storj.NodeID) (piecesTotal int64, piecesContentSize int64, pieceCount int64, err error)

WalkAndComputeSpaceUsedBySatellite returns the total used space by satellite.

func (*Supervisor) WalkCleanupTrash

func (fw *Supervisor) WalkCleanupTrash(ctx context.Context, satelliteID storj.NodeID, dateBefore time.Time) (bytesDeleted int64, keysDeleted []storj.PieceID, err error)

WalkCleanupTrash deletes per-day trash directories which are older than the given time. The lazyfilewalker does not update the space used by the trash so the caller should update the space used after the filewalker completes.

func (*Supervisor) WalkSatellitePiecesToTrash

func (fw *Supervisor) WalkSatellitePiecesToTrash(ctx context.Context, satelliteID storj.NodeID, createdBefore time.Time, filter *bloomfilter.Filter, trashFunc func(pieceID storj.PieceID) error) (piecesCount, piecesSkipped int64, err error)

WalkSatellitePiecesToTrash walks the satellite pieces and moves the pieces that are trash to the trash using the trashFunc provided.

type TrashCleanupRequest

type TrashCleanupRequest struct {
	SatelliteID storj.NodeID `json:"satelliteID"`
	DateBefore  time.Time    `json:"dateBefore"`
}

TrashCleanupRequest is the request struct for the trash-cleanup-filewalker process.

type TrashCleanupResponse

type TrashCleanupResponse struct {
	BytesDeleted int64           `json:"bytesDeleted"`
	KeysDeleted  []storj.PieceID `json:"keysDeleted"`
}

TrashCleanupResponse is the response struct for the trash-cleanup-filewalker process.

type TrashHandler

type TrashHandler struct {
	// contains filtered or unexported fields
}

TrashHandler is a writer that processes the output of the gc-filewalker subprocess.

func NewTrashHandler

func NewTrashHandler(log *zap.Logger, trashFunc func(pieceID storj.PieceID) error) *TrashHandler

NewTrashHandler creates new trash handler.

func (*TrashHandler) Decode

func (t *TrashHandler) Decode(v interface{}) error

Decode decodes the data from the buffer into the provided value.

func (*TrashHandler) Write

func (t *TrashHandler) Write(b []byte) (n int, err error)

Write writes the provided bytes to the buffer.

type UsedSpaceRequest

type UsedSpaceRequest struct {
	SatelliteID storj.NodeID `json:"satelliteID"`
}

UsedSpaceRequest is the request struct for the used-space-filewalker process.

type UsedSpaceResponse

type UsedSpaceResponse struct {
	PiecesTotal       int64 `json:"piecesTotal"`
	PiecesContentSize int64 `json:"piecesContentSize"`
	PieceCount        int64 `json:"pieceCount"`
}

UsedSpaceResponse is the response struct for the used-space-filewalker process.