fs

package
v1.3.16 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 31, 2023 License: MIT Imports: 32 Imported by: 2

Documentation

Overview

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2021-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package fs provides mountpath and FQN abstractions and methods to resolve/map stored content

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source
const (
	ObjectType   = "ob"
	WorkfileType = "wk"
	ECSliceType  = "ec"
	ECMetaType   = "mt"
)
View Source
const (
	SieMpathIDMismatch = (1 + iota) * 10
	SieTargetIDMismatch
	SieNotEqVMD
	SieMetaCorrupted
	SieFsDiffers
	SieMpathNotFound
)
View Source
const (
	// prefixes for workfiles created by various services
	WorkfileRemote       = "remote"         // getting object from neighbor target when rebalancing
	WorkfileColdget      = "cold"           // object GET: coldget
	WorkfilePut          = "put"            // object PUT
	WorkfileCopy         = "copy"           // copy object
	WorkfileAppend       = "append"         // APPEND to object (as file)
	WorkfileAppendToArch = "append-to-arch" // APPEND to existing archive
	WorkfileCreateArch   = "create-arch"    // CREATE multi-object archive
)
View Source
const (
	FlagBeingDisabled uint64 = 1 << iota
	FlagBeingDetached
)

enum Mountpath.Flags

Variables

View Source
var CSM *contentSpecMgr

Functions

func AllMpathBcks

func AllMpathBcks(opts *WalkOpts) (bcks []cmn.Bck, err error)

func CapStatusGetWhat

func CapStatusGetWhat() (fsInfo apc.CapacityInfo)

func CountPersisted

func CountPersisted(fname string) (cnt int)

func CreateBucket

func CreateBucket(bck *cmn.Bck, nilbmd bool) (errs []error)

func Decommission

func Decommission(mdOnly bool)

func DestroyBucket

func DestroyBucket(op string, bck *cmn.Bck, bid uint64) (err error)

func DirectOpen

func DirectOpen(path string, flag int, perm os.FileMode) (*os.File, error)

DirectOpen opens a file with direct disk access (with OS caching disabled).

func FillDiskStats

func FillDiskStats(m ios.AllDiskStats)

func Get

func Get() (MPI, MPI)

returns both available and disabled mountpaths (compare with GetAvail)

func GetAllMpathUtils

func GetAllMpathUtils() (utils *ios.MpathUtil)

`ios` delegations

func GetMpathUtil

func GetMpathUtil(mpath string) int64

func GetTotalDisksSize

func GetTotalDisksSize() (uint64, error)

func GetXattr

func GetXattr(fqn, attrName string) ([]byte, error)

GetXattr gets xattr by name - see also the buffered version below

func GetXattrBuf

func GetXattrBuf(fqn, attrName string, buf []byte) (b []byte, err error)

GetXattr gets xattr by name via provided buffer

func IsDirEmpty

func IsDirEmpty(dir string) (names []string, empty bool, err error)

func LcacheIdx

func LcacheIdx(digest uint64) int

func LoadNodeID

func LoadNodeID(mpaths cos.StrSet) (mDaeID string, err error)

traverses all mountpaths to load and validate node ID

func MarkerExists

func MarkerExists(marker string) bool

func MountpathsToLists

func MountpathsToLists() (mpl *apc.MountpathList)

func New

func New(num int, allowSharedDisksAndNoDisks bool)

create a new singleton

func NumAvail

func NumAvail() int

Returns number of available mountpaths

func PersistMarker

func PersistMarker(marker string) (fatalErr, writeErr error)

func PersistOnMpaths

func PersistOnMpaths(fname, backupName string, meta jsp.Opts, atMost int, b []byte, sgl *memsys.SGL) (cnt, availCnt int)

PersistOnMpaths persists `what` on mountpaths under "mountpath.Path/path" filename. It does it on maximum `atMost` mountPaths. If `atMost == 0`, it does it on every mountpath. If `backupPath != ""`, it removes files from `backupPath` and moves files from `path` to `backupPath`. Returns how many times it has successfully stored a file.

func PutMPI

func PutMPI(available, disabled MPI)

func RemoveDaemonIDs

func RemoveDaemonIDs()

func RemoveMarker

func RemoveMarker(marker string) (err error)

func RenameBucketDirs

func RenameBucketDirs(bidFrom uint64, bckFrom, bckTo *cmn.Bck) (err error)

func SetXattr

func SetXattr(fqn, attrName string, data []byte) (err error)

SetXattr sets xattr name = value

func TestDisableValidation

func TestDisableValidation()

TestDisableValidation disables fsid checking and allows mountpaths without disks (testing-only)

func TestNew

func TestNew(iostater ios.IOS)

used only in tests

func ValidateNCopies

func ValidateNCopies(tname string, copies int) (err error)

func Walk

func Walk(opts *WalkOpts) error

func WalkBck

func WalkBck(opts *WalkBckOpts) error

func WalkDir

func WalkDir(dir string, ucb func(string, DirEntry) error) error

NOTE: using Go filepath.WalkDir pros: lexical deterministic order; cons: reads the entire directory

Types

type CDF added in v1.3.16

type CDF struct {
	Capacity
	Disks []string `json:"disks"` // owned disks (ios.FsDisks map => slice)
	FS    string   `json:"fs"`    // cos.Fs + cos.FsID
}

Capacity, Disks, Filesystem (CDF) (not to be confused with Cumulative Distribution Function)

type CapStatus

type CapStatus struct {
	Err        error
	TotalUsed  uint64 // bytes
	TotalAvail uint64 // bytes
	PctAvg     int32  // used average (%)
	PctMax     int32  // max used (%)
	OOS        bool
}

func Cap added in v1.3.16

func Cap() (cs CapStatus)

func CapPeriodic

func CapPeriodic(now int64, config *cmn.Config, tcdf *TargetCDF) (cs CapStatus, updated bool, err error)

NOTE: Is called only and exclusively by `stats.Trunner` providing

`config.Periodic.StatsTime` tick.

func CapRefresh added in v1.3.16

func CapRefresh(config *cmn.Config, tcdf *TargetCDF) (cs CapStatus, err error)

func (*CapStatus) IsNil

func (cs *CapStatus) IsNil() bool

func (*CapStatus) String

func (cs *CapStatus) String() (str string)

type Capacity

type Capacity struct {
	Used    uint64 `json:"used,string"`  // bytes
	Avail   uint64 `json:"avail,string"` // ditto
	PctUsed int32  `json:"pct_used"`     // %% used (redundant ok)
}

type ContentInfo

type ContentInfo struct {
	Dir  string // original directory
	Base string // original basename
	Type string // content type
	Old  bool   // true if old (subj. to space cleanup)
}

type ContentResolver

type ContentResolver interface {
	// When set to true, services like rebalance have permission to move
	// content for example to another target because it is misplaced (HRW).
	PermToMove() bool
	// When set to true, services like LRU have permission to evict/delete content
	PermToEvict() bool
	// When set to true, content can be checksumed, shown or processed in other ways.
	PermToProcess() bool

	// Generates unique base name for original one. This function may add
	// additional information to the base name.
	// prefix - user-defined marker
	GenUniqueFQN(base, prefix string) (ufqn string)
	// Parses generated unique fqn to the original one.
	ParseUniqueFQN(base string) (orig string, old, ok bool)
}

type DirEntry

type DirEntry interface {
	IsDir() bool
}

type ECMetaContentResolver

type ECMetaContentResolver struct{}

func (*ECMetaContentResolver) GenUniqueFQN

func (*ECMetaContentResolver) GenUniqueFQN(base, _ string) string

func (*ECMetaContentResolver) ParseUniqueFQN

func (*ECMetaContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)

func (*ECMetaContentResolver) PermToEvict

func (*ECMetaContentResolver) PermToEvict() bool

func (*ECMetaContentResolver) PermToMove

func (*ECMetaContentResolver) PermToMove() bool

func (*ECMetaContentResolver) PermToProcess

func (*ECMetaContentResolver) PermToProcess() bool

type ECSliceContentResolver

type ECSliceContentResolver struct{}

func (*ECSliceContentResolver) GenUniqueFQN

func (*ECSliceContentResolver) GenUniqueFQN(base, _ string) string

func (*ECSliceContentResolver) ParseUniqueFQN

func (*ECSliceContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)

func (*ECSliceContentResolver) PermToEvict

func (*ECSliceContentResolver) PermToEvict() bool

func (*ECSliceContentResolver) PermToMove

func (*ECSliceContentResolver) PermToMove() bool

func (*ECSliceContentResolver) PermToProcess

func (*ECSliceContentResolver) PermToProcess() bool

type ErrMountpathNoDisks

type ErrMountpathNoDisks struct {
	Mi *Mountpath
}

func (*ErrMountpathNoDisks) Error

func (e *ErrMountpathNoDisks) Error() string

type ErrStorageIntegrity

type ErrStorageIntegrity struct {
	Msg  string
	Code int // Sie* enum above
}

func (*ErrStorageIntegrity) Error

func (sie *ErrStorageIntegrity) Error() string

type MPI

type MPI map[string]*Mountpath

func GetAvail

func GetAvail() MPI

func (MPI) String

func (mpi MPI) String() string

type MountedFS

type MountedFS struct {
	// contains filtered or unexported fields
}

MountedFS holds all mountpaths for the target.

type Mountpath added in v1.3.16

type Mountpath struct {
	Path   string   // clean path
	cos.FS          // underlying filesystem
	Disks  []string // owned disks (ios.FsDisks map => slice)

	PathDigest uint64 // (HRW logic)
	// contains filtered or unexported fields
}

func Add

func Add(mpath, tid string) (mi *Mountpath, err error)

(used only in _unit_ tests - compare with AddMpath below)

func AddMpath

func AddMpath(mpath, tid string, cb func(), force bool) (mi *Mountpath, err error)

Add adds new mountpath to the target's `availablePaths` TODO: extend `force=true` to disregard "filesystem sharing"

func BeginDD

func BeginDD(action string, flags uint64, mpath string) (mi *Mountpath, numAvail int, err error)

begin (disable | detach) transaction: CoW-mark the corresponding mountpath

func Disable

func Disable(mpath string, cb ...func()) (disabledMpath *Mountpath, err error)

Disables a mountpath, i.e., removes it from usage but keeps in the volume (for possible future re-enablement). If successful, returns the disabled mountpath. Otherwise, returns nil (also in the case if the mountpath was already disabled).

func Enable

func Enable(mpath string) (enabledMpath *Mountpath, err error)

(used only in tests - compare with EnableMpath below)

func EnableMpath

func EnableMpath(mpath, tid string, cb func()) (enabledMpath *Mountpath, err error)

Enable enables previously disabled mountpath. enabled is set to true if mountpath has been moved from disabled to available and exists is set to true if such mountpath even exists.

func FQN2Mpath

func FQN2Mpath(fqn string) (found *Mountpath, relativePath string, err error)

FQN2Mpath matches FQN to mountpath and returns the mountpath and the relative path.

func NewMountpath

func NewMountpath(mpath string) (mi *Mountpath, err error)

func Path2Mpath

func Path2Mpath(path string) (found *Mountpath, err error)

Path2Mpath takes in any file path (e.g., ../../a/b/c) and returns the matching `mi`, if exists

func Remove

func Remove(mpath string, cb ...func()) (*Mountpath, error)

Remove removes mountpaths from the target's mountpaths. It searches for the mountpath in `available` and, if not found, in `disabled`.

func (*Mountpath) AddDisabled added in v1.3.16

func (mi *Mountpath) AddDisabled(disabledPaths MPI)

func (*Mountpath) AddEnabled added in v1.3.16

func (mi *Mountpath) AddEnabled(tid string, availablePaths MPI, config *cmn.Config) (err error)

func (*Mountpath) CheckDisks added in v1.3.16

func (mi *Mountpath) CheckDisks() (err error)

func (*Mountpath) ClearDD added in v1.3.16

func (mi *Mountpath) ClearDD()

func (*Mountpath) ClearMDs added in v1.3.16

func (mi *Mountpath) ClearMDs()

func (*Mountpath) CreateMissingBckDirs added in v1.3.16

func (mi *Mountpath) CreateMissingBckDirs(bck *cmn.Bck) (err error)

func (*Mountpath) DeletedRoot added in v1.3.16

func (mi *Mountpath) DeletedRoot() string

func (*Mountpath) EvictLomCache added in v1.3.16

func (mi *Mountpath) EvictLomCache()

func (*Mountpath) IsAnySet added in v1.3.16

func (mi *Mountpath) IsAnySet(flags uint64) bool

func (*Mountpath) IsIdle added in v1.3.16

func (mi *Mountpath) IsIdle(config *cmn.Config) bool

func (*Mountpath) LomCache added in v1.3.16

func (mi *Mountpath) LomCache(idx int) *sync.Map

func (*Mountpath) MakePathBck added in v1.3.16

func (mi *Mountpath) MakePathBck(bck *cmn.Bck) string

func (*Mountpath) MakePathCT added in v1.3.16

func (mi *Mountpath) MakePathCT(bck *cmn.Bck, contentType string) string

func (*Mountpath) MakePathFQN added in v1.3.16

func (mi *Mountpath) MakePathFQN(bck *cmn.Bck, contentType, objName string) string

func (*Mountpath) MoveToDeleted added in v1.3.16

func (mi *Mountpath) MoveToDeleted(dir string) (err error)

MoveToDeleted removes directory in steps: 1. Synchronously gets temporary directory name 2. Synchronously renames old folder to temporary directory

func (*Mountpath) Remove added in v1.3.16

func (mi *Mountpath) Remove(path string) error

func (*Mountpath) RemoveDeleted added in v1.3.16

func (mi *Mountpath) RemoveDeleted(who string) (rerr error)

func (*Mountpath) SetDaemonIDXattr added in v1.3.16

func (mi *Mountpath) SetDaemonIDXattr(tid string) error

func (*Mountpath) String added in v1.3.16

func (mi *Mountpath) String() string

func (*Mountpath) TempDir added in v1.3.16

func (mi *Mountpath) TempDir(dir string) string

type ObjectContentResolver

type ObjectContentResolver struct{}

func (*ObjectContentResolver) GenUniqueFQN

func (*ObjectContentResolver) GenUniqueFQN(base, _ string) string

func (*ObjectContentResolver) ParseUniqueFQN

func (*ObjectContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)

func (*ObjectContentResolver) PermToEvict

func (*ObjectContentResolver) PermToEvict() bool

func (*ObjectContentResolver) PermToMove

func (*ObjectContentResolver) PermToMove() bool

func (*ObjectContentResolver) PermToProcess

func (*ObjectContentResolver) PermToProcess() bool

type ParsedFQN

type ParsedFQN struct {
	Mountpath   *Mountpath
	ContentType string
	Bck         cmn.Bck
	ObjName     string
	Digest      uint64
}

func ParseFQN

func ParseFQN(fqn string) (parsed ParsedFQN, err error)

ParseFQN splits a provided FQN (created by `MakePathFQN`) or reports an error.

type PartsFQN

type PartsFQN interface {
	ObjectName() string
	Bucket() *cmn.Bck
	Mountpath() *Mountpath
	CacheIdx() int
}

type TargetCDF added in v1.3.16

type TargetCDF struct {
	Mountpaths map[string]*CDF // mpath => [Capacity, Disks, FS (CDF)]
	PctMax     int32           `json:"pct_max"` // max used (%)
	PctAvg     int32           `json:"pct_avg"` // avg used (%)
	CsErr      string          `json:"cs_err"`  // OOS or high-wm error message
}

Target (cumulative) CDF

type WalkBckOpts

type WalkBckOpts struct {
	ValidateCallback walkFunc // should return filepath.SkipDir to skip directory without an error
	WalkOpts
}

type WalkOpts

type WalkOpts struct {
	Mi       *Mountpath
	Callback walkFunc
	Bck      cmn.Bck
	Dir      string
	CTs      []string
	Sorted   bool
}

type WorkfileContentResolver

type WorkfileContentResolver struct{}

func (*WorkfileContentResolver) GenUniqueFQN

func (*WorkfileContentResolver) GenUniqueFQN(base, prefix string) string

func (*WorkfileContentResolver) ParseUniqueFQN

func (*WorkfileContentResolver) ParseUniqueFQN(base string) (orig string, old, ok bool)

func (*WorkfileContentResolver) PermToEvict

func (*WorkfileContentResolver) PermToEvict() bool

func (*WorkfileContentResolver) PermToMove

func (*WorkfileContentResolver) PermToMove() bool

func (*WorkfileContentResolver) PermToProcess

func (*WorkfileContentResolver) PermToProcess() bool

Directories

Path Synopsis
Package health provides a basic mountpath health monitor.
Package health provides a basic mountpath health monitor.
Package mpather provides per-mountpath concepts.
Package mpather provides per-mountpath concepts.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL