meta

package
v0.0.0-...-0546637 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 8, 2024 License: Apache-2.0 Imports: 63 Imported by: 0

Documentation

Overview

nolint

Index

Constants

View Source 
const (
	// MaxVersion is the max of supported versions.
	MaxVersion = 1
	// ChunkSize is size of a chunk
	ChunkSize = 1 << 26 // 64M
	// DeleteSlice is a message to delete a slice from object store.
	DeleteSlice = 1000
	// CompactChunk is a message to compact a chunk in object store.
	CompactChunk = 1001
	// Rmr is a message to remove a directory recursively.
	Rmr = 1002
	// LegacyInfo is a message to get the internal info for file or directory.
	LegacyInfo = 1003
	// FillCache is a message to build cache for target directories/files
	FillCache = 1004
	// InfoV2 is a message to get the internal info for file or directory.
	InfoV2 = 1005
	// Clone is a message to clone a file or dir from another.
	Clone = 1006
	// OpSummary is a message to get tree summary of directories.
	OpSummary = 1007
)
View Source 
const (
	TypeFile      = 1 // type for regular file
	TypeDirectory = 2 // type for directory
	TypeSymlink   = 3 // type for symlink
	TypeFIFO      = 4 // type for FIFO node
	TypeBlockDev  = 5 // type for block device
	TypeCharDev   = 6 // type for character device
	TypeSocket    = 7 // type for socket
)
View Source 
const (
	RenameNoReplace = 1 << iota
	RenameExchange
	RenameWhiteout

	RenameRestore // internal
)
View Source 
const (
	// SetAttrMode is a mask to update a attribute of node
	SetAttrMode = 1 << iota
	SetAttrUID
	SetAttrGID
	SetAttrSize
	SetAttrAtime
	SetAttrMtime
	SetAttrCtime
	SetAttrAtimeNow
	SetAttrMtimeNow
	SetAttrFlag = 1 << 15
)
View Source 
const (
	FlagImmutable = 1 << iota
	FlagAppend
)
View Source 
const (
	QuotaSet uint8 = iota
	QuotaGet
	QuotaDel
	QuotaList
	QuotaCheck
)
View Source 
const (
	// clone mode
	CLONE_MODE_CAN_OVERWRITE      = 0x01
	CLONE_MODE_PRESERVE_ATTR      = 0x02
	CLONE_MODE_PRESERVE_HARDLINKS = 0x08

	// atime mode
	NoAtime     = "noatime"
	RelAtime    = "relatime"
	StrictAtime = "strictatime"
)
View Source 
const (
	MODE_MASK_R = 0b100
	MODE_MASK_W = 0b010
	MODE_MASK_X = 0b001
)
View Source 
const (
	F_UNLCK = syscall.F_UNLCK
	F_RDLCK = syscall.F_RDLCK
	F_WRLCK = syscall.F_WRLCK
)
View Source 
const (
	XattrCreateOrReplace = 0
	XattrCreate          = sys.XATTR_CREATE
	XattrReplace         = sys.XATTR_REPLACE
)
View Source 
const CDATA = 0xFF // 4 bytes: data length
View Source 
const CPROGRESS = 0xFE // 16 bytes: progress increment

Type of control messages

View Source 
const ENOATTR = syscall.ENODATA
View Source 
const MaxFieldsCountOfTable = 16 // node table
View Source 
const MaxName = 255

Variables

View Source 
var CHARS = []byte("0123456789ABCDEF")
View Source 
var TrashName = ".trash"

Functions

func Register 

func Register(name string, register Creator)

Types

type Attr 

type Attr struct {
	Flags     uint8  // flags
	Typ       uint8  // type of a node
	Mode      uint16 // permission mode
	Uid       uint32 // owner id
	Gid       uint32 // group id of owner
	Rdev      uint32 // device number
	Atime     int64  // last access time
	Mtime     int64  // last modified time
	Ctime     int64  // last change time for meta
	Atimensec uint32 // nanosecond part of atime
	Mtimensec uint32 // nanosecond part of mtime
	Ctimensec uint32 // nanosecond part of ctime
	Nlink     uint32 // number of links (sub-directories or hardlinks)
	Length    uint64 // length of regular file

	Parent    Ino  // inode of parent; 0 means tracked by parentKey (for hardlinks)
	Full      bool // the attributes are completed or not
	KeepCache bool // whether to keep the cached page or not
}

Attr represents attributes of a node.

func (Attr) SMode 

func (a Attr) SMode() uint32

SMode is the file mode including type and unix permission.

type Config 

type Config struct {
	Strict             bool // update ctime
	Retries            int
	MaxDeletes         int
	SkipDirNlink       int
	CaseInsensi        bool
	ReadOnly           bool
	NoBGJob            bool // disable background jobs like clean-up, backup, etc.
	OpenCache          time.Duration
	OpenCacheLimit     uint64 // max number of files to cache (soft limit)
	Heartbeat          time.Duration
	MountPoint         string
	Subdir             string
	AtimeMode          string
	DirStatFlushPeriod time.Duration
}

Config for clients.

func DefaultConf 

func DefaultConf() *Config

func (*Config) SelfCheck 

func (c *Config) SelfCheck()

type Context 

type Context interface {
	context.Context
	Gid() uint32
	Gids() []uint32
	Uid() uint32
	Pid() uint32
	WithValue(k, v interface{})
	Cancel()
	Canceled() bool
	CheckPermission() bool
}
var Background Context = WrapContext(context.Background())

func NewContext 

func NewContext(pid, uid uint32, gids []uint32) Context

func WrapContext 

func WrapContext(ctx context.Context) Context

type Creator 

type Creator func(driver, addr string, conf *Config) (Meta, error)

type CtxKey 

type CtxKey string

type DumpedAttr 

type DumpedAttr struct {
	Inode     Ino    `json:"inode"`
	Flags     uint8  `json:"flags,omitempty"`
	Type      string `json:"type"`
	Mode      uint16 `json:"mode"`
	Uid       uint32 `json:"uid"`
	Gid       uint32 `json:"gid"`
	Atime     int64  `json:"atime"`
	Mtime     int64  `json:"mtime"`
	Ctime     int64  `json:"ctime"`
	Atimensec uint32 `json:"atimensec,omitempty"`
	Mtimensec uint32 `json:"mtimensec,omitempty"`
	Ctimensec uint32 `json:"ctimensec,omitempty"`
	Nlink     uint32 `json:"nlink"`
	Length    uint64 `json:"length"`
	Rdev      uint32 `json:"rdev,omitempty"`
	// contains filtered or unexported fields
}

type DumpedChunk 

type DumpedChunk struct {
	Index  uint32         `json:"index"`
	Slices []*DumpedSlice `json:"slices"`
}

type DumpedCounters 

type DumpedCounters struct {
	UsedSpace         int64 `json:"usedSpace"`
	UsedInodes        int64 `json:"usedInodes"`
	NextInode         int64 `json:"nextInodes"`
	NextChunk         int64 `json:"nextChunk"`
	NextSession       int64 `json:"nextSession"`
	NextTrash         int64 `json:"nextTrash"`
	NextCleanupSlices int64 `json:"nextCleanupSlices,omitempty"` // deprecated, always 0
}

type DumpedDelFile 

type DumpedDelFile struct {
	Inode  Ino    `json:"inode"`
	Length uint64 `json:"length"`
	Expire int64  `json:"expire"`
}

type DumpedEntry 

type DumpedEntry struct {
	Name    string                  `json:"-"`
	Parents []Ino                   `json:"-"`
	Attr    *DumpedAttr             `json:"attr,omitempty"`
	Symlink string                  `json:"symlink,omitempty"`
	Xattrs  []*DumpedXattr          `json:"xattrs,omitempty"`
	Chunks  []*DumpedChunk          `json:"chunks,omitempty"`
	Entries map[string]*DumpedEntry `json:"entries,omitempty"`
}

type DumpedMeta 

type DumpedMeta struct {
	Setting   Format
	Counters  *DumpedCounters
	Sustained []*DumpedSustained
	DelFiles  []*DumpedDelFile
	Quotas    map[Ino]*DumpedQuota `json:",omitempty"`
	FSTree    *DumpedEntry         `json:",omitempty"`
	Trash     *DumpedEntry         `json:",omitempty"`
}

type DumpedQuota 

type DumpedQuota struct {
	MaxSpace   int64 `json:"maxSpace"`
	MaxInodes  int64 `json:"maxInodes"`
	UsedSpace  int64 `json:"-"`
	UsedInodes int64 `json:"-"`
}

type DumpedSlice 

type DumpedSlice struct {
	Chunkid uint64 `json:"chunkid,omitempty"`
	Id      uint64 `json:"id"`
	Pos     uint32 `json:"pos,omitempty"`
	Size    uint32 `json:"size"`
	Off     uint32 `json:"off,omitempty"`
	Len     uint32 `json:"len"`
}

type DumpedSustained 

type DumpedSustained struct {
	Sid    uint64 `json:"sid"`
	Inodes []Ino  `json:"inodes"`
}

type DumpedXattr 

type DumpedXattr struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

type Entry 

type Entry struct {
	Inode Ino
	Name  []byte
	Attr  *Attr
}

Entry is an entry inside a directory.

type FLockItem 

type FLockItem struct {
	Type string
	// contains filtered or unexported fields
}

type Flock 

type Flock struct {
	Inode Ino
	Owner uint64
	Ltype string
}

type Format 

type Format struct {
	Name             string
	UUID             string
	Storage          string
	StorageClass     string `json:",omitempty"`
	Bucket           string
	AccessKey        string `json:",omitempty"`
	SecretKey        string `json:",omitempty"`
	SessionToken     string `json:",omitempty"`
	BlockSize        int
	Compression      string `json:",omitempty"`
	Shards           int    `json:",omitempty"`
	HashPrefix       bool   `json:",omitempty"`
	Capacity         uint64 `json:",omitempty"`
	Inodes           uint64 `json:",omitempty"`
	EncryptKey       string `json:",omitempty"`
	EncryptAlgo      string `json:",omitempty"`
	KeyEncrypted     bool   `json:",omitempty"`
	UploadLimit      int64  `json:",omitempty"` // Mbps
	DownloadLimit    int64  `json:",omitempty"` // Mbps
	TrashDays        int
	MetaVersion      int    `json:",omitempty"`
	MinClientVersion string `json:",omitempty"`
	MaxClientVersion string `json:",omitempty"`
	DirStats         bool   `json:",omitempty"`
}

func (*Format) CheckVersion 

func (f *Format) CheckVersion() error

func (*Format) Decrypt 

func (f *Format) Decrypt() error

func (*Format) Encrypt 

func (f *Format) Encrypt() error

func (*Format) RemoveSecret 

func (f *Format) RemoveSecret()

func (*Format) String 

func (f *Format) String() string

type Ino 

type Ino uint64
const RootInode Ino = 1
const TrashInode Ino = 0x7FFFFFFF10000000 // larger than vfs.minInternalNode

func (Ino) String 

func (i Ino) String() string

type Meta 

type Meta interface {
	// Name of database
	Name() string
	// Init is used to initialize a meta service.
	Init(format *Format, force bool) error
	// Shutdown close current database connections.
	Shutdown() error
	// Reset cleans up all metadata, VERY DANGEROUS!
	Reset() error
	// Load loads the existing setting of a formatted volume from meta service.
	Load(checkVersion bool) (*Format, error)
	// NewSession creates a new client session.
	NewSession(record bool) error
	// CloseSession does cleanup and close the session.
	CloseSession() error
	// GetSession retrieves information of session with sid
	GetSession(sid uint64, detail bool) (*Session, error)
	// ListSessions returns all client sessions.
	ListSessions() ([]*Session, error)
	// ScanDeletedObject scan deleted objects by customized scanner.
	ScanDeletedObject(Context, trashSliceScan, pendingSliceScan, trashFileScan, pendingFileScan) error
	// ListLocks returns all locks of a inode.
	ListLocks(ctx context.Context, inode Ino) ([]PLockItem, []FLockItem, error)
	// CleanStaleSessions cleans up sessions not active for more than 5 minutes
	CleanStaleSessions()
	// CleanupTrashBefore deletes all files in trash before the given time.
	CleanupTrashBefore(ctx Context, edge time.Time, increProgress func(int))
	// CleanupDetachedNodesBefore deletes all detached nodes before the given time.
	CleanupDetachedNodesBefore(ctx Context, edge time.Time, increProgress func())

	// StatFS returns summary statistics of a volume.
	StatFS(ctx Context, ino Ino, totalspace, availspace, iused, iavail *uint64) syscall.Errno
	// Access checks the access permission on given inode.
	Access(ctx Context, inode Ino, modemask uint8, attr *Attr) syscall.Errno
	// Lookup returns the inode and attributes for the given entry in a directory.
	Lookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr, checkPerm bool) syscall.Errno
	// Resolve fetches the inode and attributes for an entry identified by the given path.
	// ENOTSUP will be returned if there's no natural implementation for this operation or
	// if there are any symlink following involved.
	Resolve(ctx Context, parent Ino, path string, inode *Ino, attr *Attr) syscall.Errno
	// GetAttr returns the attributes for given node.
	GetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno
	// SetAttr updates the attributes for given node.
	SetAttr(ctx Context, inode Ino, set uint16, sggidclearmode uint8, attr *Attr) syscall.Errno
	// Check setting attr is allowed or not
	CheckSetAttr(ctx Context, inode Ino, set uint16, attr Attr) syscall.Errno
	// Truncate changes the length for given file.
	Truncate(ctx Context, inode Ino, flags uint8, attrlength uint64, attr *Attr, skipPermCheck bool) syscall.Errno
	// Fallocate preallocate given space for given file.
	Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64) syscall.Errno
	// ReadLink returns the target of a symlink.
	ReadLink(ctx Context, inode Ino, path *[]byte) syscall.Errno
	// Symlink creates a symlink in a directory with given name.
	Symlink(ctx Context, parent Ino, name string, path string, inode *Ino, attr *Attr) syscall.Errno
	// Mknod creates a node in a directory with given name, type and permissions.
	Mknod(ctx Context, parent Ino, name string, _type uint8, mode uint16, cumask uint16, rdev uint32, path string, inode *Ino, attr *Attr) syscall.Errno
	// Mkdir creates a sub-directory with given name and mode.
	Mkdir(ctx Context, parent Ino, name string, mode uint16, cumask uint16, copysgid uint8, inode *Ino, attr *Attr) syscall.Errno
	// Unlink removes a file entry from a directory.
	// The file will be deleted if it's not linked by any entries and not open by any sessions.
	Unlink(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno
	// Rmdir removes an empty sub-directory.
	Rmdir(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno
	// Rename move an entry from a source directory to another with given name.
	// The targeted entry will be overwrited if it's a file or empty directory.
	// For Hadoop, the target should not be overwritten.
	Rename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode *Ino, attr *Attr) syscall.Errno
	// Link creates an entry for node.
	Link(ctx Context, inodeSrc, parent Ino, name string, attr *Attr) syscall.Errno
	// Readdir returns all entries for given directory, which include attributes if plus is true.
	Readdir(ctx Context, inode Ino, wantattr uint8, entries *[]*Entry) syscall.Errno
	// Create creates a file in a directory with given name.
	Create(ctx Context, parent Ino, name string, mode uint16, cumask uint16, flags uint32, inode *Ino, attr *Attr) syscall.Errno
	// Open checks permission on a node and track it as open.
	Open(ctx Context, inode Ino, flags uint32, attr *Attr) syscall.Errno
	// Close a file.
	Close(ctx Context, inode Ino) syscall.Errno
	// Read returns the list of slices on the given chunk.
	Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) syscall.Errno
	// NewSlice returns an id for new slice.
	NewSlice(ctx Context, id *uint64) syscall.Errno
	// Write put a slice of data on top of the given chunk.
	Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time) syscall.Errno
	// InvalidateChunkCache invalidate chunk cache
	InvalidateChunkCache(ctx Context, inode Ino, indx uint32) syscall.Errno
	// CopyFileRange copies part of a file to another one.
	CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied *uint64) syscall.Errno
	// GetParents returns a map of node parents (> 1 parents if hardlinked)
	GetParents(ctx Context, inode Ino) map[Ino]int
	// GetDirStat returns the space and inodes usage of a directory.
	GetDirStat(ctx Context, inode Ino) (stat *dirStat, st syscall.Errno)

	// GetXattr returns the value of extended attribute for given name.
	GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno
	// ListXattr returns all extended attributes of a node.
	ListXattr(ctx Context, inode Ino, dbuff *[]byte) syscall.Errno
	// SetXattr update the extended attribute of a node.
	SetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno
	// RemoveXattr removes the extended attribute of a node.
	RemoveXattr(ctx Context, inode Ino, name string) syscall.Errno
	// Flock tries to put a lock on given file.
	Flock(ctx Context, inode Ino, owner uint64, ltype uint32, block bool) syscall.Errno
	// Getlk returns the current lock owner for a range on a file.
	Getlk(ctx Context, inode Ino, owner uint64, ltype *uint32, start, end *uint64, pid *uint32) syscall.Errno
	// Setlk sets a file range lock on given file.
	Setlk(ctx Context, inode Ino, owner uint64, block bool, ltype uint32, start, end uint64, pid uint32) syscall.Errno

	// Compact all the chunks by merge small slices together
	CompactAll(ctx Context, threads int, bar *utils.Bar) syscall.Errno
	// ListSlices returns all slices used by all files.
	ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno
	// Remove all files and directories recursively.
	Remove(ctx Context, parent Ino, name string, count *uint64) syscall.Errno
	// Get summary of a node; for a directory it will accumulate all its child nodes
	GetSummary(ctx Context, inode Ino, summary *Summary, recursive bool, strict bool) syscall.Errno
	// GetTreeSummary returns a summary in tree structure
	GetTreeSummary(ctx Context, root *TreeSummary, depth, topN uint8, strict bool, updateProgress func(count uint64, bytes uint64)) syscall.Errno
	// Clone a file or directory
	Clone(ctx Context, srcIno, dstParentIno Ino, dstName string, cmode uint8, cumask uint16, count, total *uint64) syscall.Errno
	// GetPaths returns all paths of an inode
	GetPaths(ctx Context, inode Ino) []string
	// Check integrity of an absolute path and repair it if asked
	Check(ctx Context, fpath string, repair bool, recursive bool, statAll bool) error
	// Change root to a directory specified by subdir
	Chroot(ctx Context, subdir string) syscall.Errno

	// Get a copy of the current format
	GetFormat() Format

	// OnMsg add a callback for the given message type.
	OnMsg(mtype uint32, cb MsgCallback)
	// OnReload register a callback for any change founded after reloaded.
	OnReload(func(new *Format))

	HandleQuota(ctx Context, cmd uint8, dpath string, quotas map[string]*Quota, strict, repair bool) error

	// Dump the tree under root, which may be modified by checkRoot
	DumpMeta(w io.Writer, root Ino, keepSecret bool) error
	LoadMeta(r io.Reader) error

	InitMetrics(registerer prometheus.Registerer)
	// contains filtered or unexported methods
}

Meta is a interface for a meta service for file system.

func NewClient 

func NewClient(uri string, conf *Config) Meta

NewClient creates a Meta client for given uri.

type MsgCallback 

type MsgCallback func(...interface{}) error

MsgCallback is a callback for messages from meta service.

type PLockItem 

type PLockItem struct {
	// contains filtered or unexported fields
}

type Plock 

type Plock struct {
	Inode   Ino
	Owner   uint64
	Records []plockRecord
}

type Quota 

type Quota struct {
	MaxSpace, MaxInodes   int64
	UsedSpace, UsedInodes int64
	// contains filtered or unexported fields
}

type Session 

type Session struct {
	Sid    uint64
	Expire time.Time
	SessionInfo
	Sustained []Ino   `json:",omitempty"`
	Flocks    []Flock `json:",omitempty"`
	Plocks    []Plock `json:",omitempty"`
}

Session contains detailed information of a client session

type SessionInfo 

type SessionInfo struct {
	Version    string
	HostName   string
	IPAddrs    []string `json:",omitempty"`
	MountPoint string
	ProcessID  int
}

type Slice 

type Slice struct {
	Id   uint64
	Size uint32
	Off  uint32
	Len  uint32
}

Slice is a slice of a chunk. Multiple slices could be combined together as a chunk.

type Summary 

type Summary struct {
	Length uint64
	Size   uint64
	Files  uint64
	Dirs   uint64
}

Summary represents the total number of files/directories and total length of all files inside a directory.

type TreeSummary 

type TreeSummary struct {
	Inode    Ino
	Path     string
	Type     uint8
	Size     uint64
	Files    uint64
	Dirs     uint64
	Children []*TreeSummary `json:",omitempty"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.