meta

package
v1.2.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 30, 2024 License: Apache-2.0 Imports: 65 Imported by: 17

Documentation

Overview

nolint

Index

Constants

View Source 
const (
	// MaxVersion is the max of supported versions.
	MaxVersion = 1
	// ChunkSize is size of a chunk
	ChunkSize = 1 << 26 // 64M
	// DeleteSlice is a message to delete a slice from object store.
	DeleteSlice = 1000
	// CompactChunk is a message to compact a chunk in object store.
	CompactChunk = 1001
	// Rmr is a message to remove a directory recursively.
	Rmr = 1002
	// LegacyInfo is a message to get the internal info for file or directory.
	LegacyInfo = 1003
	// FillCache is a message to build cache for target directories/files
	FillCache = 1004
	// InfoV2 is a message to get the internal info for file or directory.
	InfoV2 = 1005
	// Clone is a message to clone a file or dir from another.
	Clone = 1006
	// OpSummary is a message to get tree summary of directories.
	OpSummary = 1007
	// CompactPath is a message to trigger compact
	CompactPath = 1008
)
View Source 
const (
	TypeFile      = 1 // type for regular file
	TypeDirectory = 2 // type for directory
	TypeSymlink   = 3 // type for symlink
	TypeFIFO      = 4 // type for FIFO node
	TypeBlockDev  = 5 // type for block device
	TypeCharDev   = 6 // type for character device
	TypeSocket    = 7 // type for socket
)
View Source 
const (
	RenameNoReplace = 1 << iota
	RenameExchange
	RenameWhiteout

	RenameRestore // internal
)
View Source 
const (
	// SetAttrMode is a mask to update a attribute of node
	SetAttrMode = 1 << iota
	SetAttrUID
	SetAttrGID
	SetAttrSize
	SetAttrAtime
	SetAttrMtime
	SetAttrCtime
	SetAttrAtimeNow
	SetAttrMtimeNow
	SetAttrFlag = 1 << 15
)
View Source 
const (
	FlagImmutable = 1 << iota
	FlagAppend
)
View Source 
const (
	QuotaSet uint8 = iota
	QuotaGet
	QuotaDel
	QuotaList
	QuotaCheck
)
View Source 
const (
	// clone mode
	CLONE_MODE_CAN_OVERWRITE      = 0x01
	CLONE_MODE_PRESERVE_ATTR      = 0x02
	CLONE_MODE_PRESERVE_HARDLINKS = 0x08

	// atime mode
	NoAtime     = "noatime"
	RelAtime    = "relatime"
	StrictAtime = "strictatime"
)
View Source 
const (
	MODE_MASK_R = 0b100
	MODE_MASK_W = 0b010
	MODE_MASK_X = 0b001
)
View Source 
const (
	F_UNLCK = syscall.F_UNLCK
	F_RDLCK = syscall.F_RDLCK
	F_WRLCK = syscall.F_WRLCK
)
View Source 
const (
	XattrCreateOrReplace = 0
	XattrCreate          = sys.XATTR_CREATE
	XattrReplace         = sys.XATTR_REPLACE
)
View Source 
const CDATA = 0xFF // 4 bytes: data length
View Source 
const CPROGRESS = 0xFE // 16 bytes: progress increment

Type of control messages

View Source 
const ENOATTR = syscall.ENODATA
View Source 
const MaxFieldsCountOfTable = 18 // node table
View Source 
const MaxName = 255
View Source 
const MaxSymlink = 4096
View Source 
const UmountCode = 11

Variables

View Source 
var CHARS = []byte("0123456789ABCDEF")
View Source 
var TrashName = ".trash"

Functions

func Register 

func Register(name string, register Creator)

Types

type Attr 

type Attr struct {
	Flags     uint8  // flags
	Typ       uint8  // type of a node
	Mode      uint16 // permission mode
	Uid       uint32 // owner id
	Gid       uint32 // group id of owner
	Rdev      uint32 // device number
	Atime     int64  // last access time
	Mtime     int64  // last modified time
	Ctime     int64  // last change time for meta
	Atimensec uint32 // nanosecond part of atime
	Mtimensec uint32 // nanosecond part of mtime
	Ctimensec uint32 // nanosecond part of ctime
	Nlink     uint32 // number of links (sub-directories or hardlinks)
	Length    uint64 // length of regular file

	Parent    Ino  // inode of parent; 0 means tracked by parentKey (for hardlinks)
	Full      bool // the attributes are completed or not
	KeepCache bool // whether to keep the cached page or not

	AccessACL  uint32 // access ACL id (identical ACL rules share the same access ACL ID.)
	DefaultACL uint32 // default ACL id (default ACL and the access ACL share the same cache and store)
}

Attr represents attributes of a node.

func (Attr) SMode 

func (a Attr) SMode() uint32

SMode is the file mode including type and unix permission.

type Config 

type Config struct {
	Strict             bool // update ctime
	Retries            int
	MaxDeletes         int
	SkipDirNlink       int
	CaseInsensi        bool
	ReadOnly           bool
	NoBGJob            bool // disable background jobs like clean-up, backup, etc.
	OpenCache          time.Duration
	OpenCacheLimit     uint64 // max number of files to cache (soft limit)
	Heartbeat          time.Duration
	MountPoint         string
	Subdir             string
	AtimeMode          string
	DirStatFlushPeriod time.Duration
	SkipDirMtime       time.Duration
	Sid                uint64
}

Config for clients.

func DefaultConf 

func DefaultConf() *Config

func (*Config) SelfCheck 

func (c *Config) SelfCheck()

type Context 

type Context interface {
	context.Context
	Gid() uint32
	Gids() []uint32
	Uid() uint32
	Pid() uint32
	WithValue(k, v interface{})
	Cancel()
	Canceled() bool
	CheckPermission() bool
}
var Background Context = WrapContext(context.Background())

func NewContext 

func NewContext(pid, uid uint32, gids []uint32) Context

func WrapContext 

func WrapContext(ctx context.Context) Context

type Creator 

type Creator func(driver, addr string, conf *Config) (Meta, error)

type CtxKey 

type CtxKey string

type DumpedACL 

type DumpedACL struct {
	Owner  uint16           `json:"owner"`
	Group  uint16           `json:"group"`
	Other  uint16           `json:"other"`
	Mask   uint16           `json:"mask"`
	Users  []DumpedACLEntry `json:"users"`
	Groups []DumpedACLEntry `json:"groups"`
}

type DumpedACLEntry 

type DumpedACLEntry struct {
	Id   uint32 `json:"id"`
	Perm uint16 `json:"perm"`
}

type DumpedAttr 

type DumpedAttr struct {
	Inode     Ino    `json:"inode"`
	Flags     uint8  `json:"flags,omitempty"`
	Type      string `json:"type"`
	Mode      uint16 `json:"mode"`
	Uid       uint32 `json:"uid"`
	Gid       uint32 `json:"gid"`
	Atime     int64  `json:"atime"`
	Mtime     int64  `json:"mtime"`
	Ctime     int64  `json:"ctime"`
	Atimensec uint32 `json:"atimensec,omitempty"`
	Mtimensec uint32 `json:"mtimensec,omitempty"`
	Ctimensec uint32 `json:"ctimensec,omitempty"`
	Nlink     uint32 `json:"nlink"`
	Length    uint64 `json:"length"`
	Rdev      uint32 `json:"rdev,omitempty"`
	// contains filtered or unexported fields
}

type DumpedChunk 

type DumpedChunk struct {
	Index  uint32         `json:"index"`
	Slices []*DumpedSlice `json:"slices"`
}

type DumpedCounters 

type DumpedCounters struct {
	UsedSpace         int64 `json:"usedSpace"`
	UsedInodes        int64 `json:"usedInodes"`
	NextInode         int64 `json:"nextInodes"`
	NextChunk         int64 `json:"nextChunk"`
	NextSession       int64 `json:"nextSession"`
	NextTrash         int64 `json:"nextTrash"`
	NextCleanupSlices int64 `json:"nextCleanupSlices,omitempty"` // deprecated, always 0
}

type DumpedDelFile 

type DumpedDelFile struct {
	Inode  Ino    `json:"inode"`
	Length uint64 `json:"length"`
	Expire int64  `json:"expire"`
}

type DumpedEntry 

type DumpedEntry struct {
	Name       string                  `json:"-"`
	Parents    []Ino                   `json:"-"`
	Attr       *DumpedAttr             `json:"attr,omitempty"`
	Symlink    string                  `json:"symlink,omitempty"`
	Xattrs     []*DumpedXattr          `json:"xattrs,omitempty"`
	Chunks     []*DumpedChunk          `json:"chunks,omitempty"`
	Entries    map[string]*DumpedEntry `json:"entries,omitempty"`
	AccessACL  *DumpedACL              `json:"posix_acl_access,omitempty"`
	DefaultACL *DumpedACL              `json:"posix_acl_default,omitempty"`
}

type DumpedMeta 

type DumpedMeta struct {
	Setting   Format
	Counters  *DumpedCounters
	Sustained []*DumpedSustained
	DelFiles  []*DumpedDelFile
	Quotas    map[Ino]*DumpedQuota `json:",omitempty"`
	FSTree    *DumpedEntry         `json:",omitempty"`
	Trash     *DumpedEntry         `json:",omitempty"`
}

type DumpedQuota 

type DumpedQuota struct {
	MaxSpace   int64 `json:"maxSpace"`
	MaxInodes  int64 `json:"maxInodes"`
	UsedSpace  int64 `json:"-"`
	UsedInodes int64 `json:"-"`
}

type DumpedSlice 

type DumpedSlice struct {
	Chunkid uint64 `json:"chunkid,omitempty"`
	Id      uint64 `json:"id"`
	Pos     uint32 `json:"pos,omitempty"`
	Size    uint32 `json:"size"`
	Off     uint32 `json:"off,omitempty"`
	Len     uint32 `json:"len"`
}

type DumpedSustained 

type DumpedSustained struct {
	Sid    uint64 `json:"sid"`
	Inodes []Ino  `json:"inodes"`
}

type DumpedXattr 

type DumpedXattr struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

type Entry 

type Entry struct {
	Inode Ino
	Name  []byte
	Attr  *Attr
}

Entry is an entry inside a directory.

type FLockItem 

type FLockItem struct {
	Type string
	// contains filtered or unexported fields
}

type Flock 

type Flock struct {
	Inode Ino
	Owner uint64
	Ltype string
}

type Format 

type Format struct {
	Name             string
	UUID             string
	Storage          string
	StorageClass     string `json:",omitempty"`
	Bucket           string
	AccessKey        string `json:",omitempty"`
	SecretKey        string `json:",omitempty"`
	SessionToken     string `json:",omitempty"`
	BlockSize        int
	Compression      string `json:",omitempty"`
	Shards           int    `json:",omitempty"`
	HashPrefix       bool   `json:",omitempty"`
	Capacity         uint64 `json:",omitempty"`
	Inodes           uint64 `json:",omitempty"`
	EncryptKey       string `json:",omitempty"`
	EncryptAlgo      string `json:",omitempty"`
	KeyEncrypted     bool   `json:",omitempty"`
	UploadLimit      int64  `json:",omitempty"` // Mbps
	DownloadLimit    int64  `json:",omitempty"` // Mbps
	TrashDays        int
	MetaVersion      int    `json:",omitempty"`
	MinClientVersion string `json:",omitempty"`
	MaxClientVersion string `json:",omitempty"`
	DirStats         bool   `json:",omitempty"`
	EnableACL        bool
}

func (*Format) CheckCliVersion 

func (f *Format) CheckCliVersion(ver *version.Semver) error

func (*Format) CheckVersion 

func (f *Format) CheckVersion() error

func (*Format) Decrypt 

func (f *Format) Decrypt() error

func (*Format) Encrypt 

func (f *Format) Encrypt() error

func (*Format) RemoveSecret 

func (f *Format) RemoveSecret()

func (*Format) String 

func (f *Format) String() string

type Ino 

type Ino uint64
const RootInode Ino = 1
const TrashInode Ino = 0x7FFFFFFF10000000 // larger than vfs.minInternalNode

func (Ino) IsNormal 

func (i Ino) IsNormal() bool

func (Ino) IsTrash 

func (i Ino) IsTrash() bool

func (Ino) IsValid 

func (i Ino) IsValid() bool

func (Ino) String 

func (i Ino) String() string

type Meta 

type Meta interface {
	// Name of database
	Name() string
	// Init is used to initialize a meta service.
	Init(format *Format, force bool) error
	// Shutdown close current database connections.
	Shutdown() error
	// Reset cleans up all metadata, VERY DANGEROUS!
	Reset() error
	// Load loads the existing setting of a formatted volume from meta service.
	Load(checkVersion bool) (*Format, error)
	// NewSession creates or update client session.
	NewSession(record bool) error
	// CloseSession does cleanup and close the session.
	CloseSession() error
	// GetSession retrieves information of session with sid
	GetSession(sid uint64, detail bool) (*Session, error)
	// ListSessions returns all client sessions.
	ListSessions() ([]*Session, error)
	// ScanDeletedObject scan deleted objects by customized scanner.
	ScanDeletedObject(Context, trashSliceScan, pendingSliceScan, trashFileScan, pendingFileScan) error
	// ListLocks returns all locks of a inode.
	ListLocks(ctx context.Context, inode Ino) ([]PLockItem, []FLockItem, error)
	// CleanStaleSessions cleans up sessions not active for more than 5 minutes
	CleanStaleSessions()
	// CleanupTrashBefore deletes all files in trash before the given time.
	CleanupTrashBefore(ctx Context, edge time.Time, increProgress func(int))
	// CleanupDetachedNodesBefore deletes all detached nodes before the given time.
	CleanupDetachedNodesBefore(ctx Context, edge time.Time, increProgress func())

	// StatFS returns summary statistics of a volume.
	StatFS(ctx Context, ino Ino, totalspace, availspace, iused, iavail *uint64) syscall.Errno
	// Access checks the access permission on given inode.
	Access(ctx Context, inode Ino, modemask uint8, attr *Attr) syscall.Errno
	// Lookup returns the inode and attributes for the given entry in a directory.
	Lookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr, checkPerm bool) syscall.Errno
	// Resolve fetches the inode and attributes for an entry identified by the given path.
	// ENOTSUP will be returned if there's no natural implementation for this operation or
	// if there are any symlink following involved.
	Resolve(ctx Context, parent Ino, path string, inode *Ino, attr *Attr) syscall.Errno
	// GetAttr returns the attributes for given node.
	GetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno
	// SetAttr updates the attributes for given node.
	SetAttr(ctx Context, inode Ino, set uint16, sggidclearmode uint8, attr *Attr) syscall.Errno
	// Check setting attr is allowed or not
	CheckSetAttr(ctx Context, inode Ino, set uint16, attr Attr) syscall.Errno
	// Truncate changes the length for given file.
	Truncate(ctx Context, inode Ino, flags uint8, attrlength uint64, attr *Attr, skipPermCheck bool) syscall.Errno
	// Fallocate preallocate given space for given file.
	Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64, length *uint64) syscall.Errno
	// ReadLink returns the target of a symlink.
	ReadLink(ctx Context, inode Ino, path *[]byte) syscall.Errno
	// Symlink creates a symlink in a directory with given name.
	Symlink(ctx Context, parent Ino, name string, path string, inode *Ino, attr *Attr) syscall.Errno
	// Mknod creates a node in a directory with given name, type and permissions.
	Mknod(ctx Context, parent Ino, name string, _type uint8, mode uint16, cumask uint16, rdev uint32, path string, inode *Ino, attr *Attr) syscall.Errno
	// Mkdir creates a sub-directory with given name and mode.
	Mkdir(ctx Context, parent Ino, name string, mode uint16, cumask uint16, copysgid uint8, inode *Ino, attr *Attr) syscall.Errno
	// Unlink removes a file entry from a directory.
	// The file will be deleted if it's not linked by any entries and not open by any sessions.
	Unlink(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno
	// Rmdir removes an empty sub-directory.
	Rmdir(ctx Context, parent Ino, name string, skipCheckTrash ...bool) syscall.Errno
	// Rename move an entry from a source directory to another with given name.
	// The targeted entry will be overwrited if it's a file or empty directory.
	// For Hadoop, the target should not be overwritten.
	Rename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode *Ino, attr *Attr) syscall.Errno
	// Link creates an entry for node.
	Link(ctx Context, inodeSrc, parent Ino, name string, attr *Attr) syscall.Errno
	// Readdir returns all entries for given directory, which include attributes if plus is true.
	Readdir(ctx Context, inode Ino, wantattr uint8, entries *[]*Entry) syscall.Errno
	// Create creates a file in a directory with given name.
	Create(ctx Context, parent Ino, name string, mode uint16, cumask uint16, flags uint32, inode *Ino, attr *Attr) syscall.Errno
	// Open checks permission on a node and track it as open.
	Open(ctx Context, inode Ino, flags uint32, attr *Attr) syscall.Errno
	// Close a file.
	Close(ctx Context, inode Ino) syscall.Errno
	// Read returns the list of slices on the given chunk.
	Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) syscall.Errno
	// NewSlice returns an id for new slice.
	NewSlice(ctx Context, id *uint64) syscall.Errno
	// Write put a slice of data on top of the given chunk.
	Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice, mtime time.Time) syscall.Errno
	// InvalidateChunkCache invalidate chunk cache
	InvalidateChunkCache(ctx Context, inode Ino, indx uint32) syscall.Errno
	// CopyFileRange copies part of a file to another one.
	CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied, outLength *uint64) syscall.Errno
	// GetParents returns a map of node parents (> 1 parents if hardlinked)
	GetParents(ctx Context, inode Ino) map[Ino]int
	// GetDirStat returns the space and inodes usage of a directory.
	GetDirStat(ctx Context, inode Ino) (stat *dirStat, st syscall.Errno)

	// GetXattr returns the value of extended attribute for given name.
	GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno
	// ListXattr returns all extended attributes of a node.
	ListXattr(ctx Context, inode Ino, dbuff *[]byte) syscall.Errno
	// SetXattr update the extended attribute of a node.
	SetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno
	// RemoveXattr removes the extended attribute of a node.
	RemoveXattr(ctx Context, inode Ino, name string) syscall.Errno
	// Flock tries to put a lock on given file.
	Flock(ctx Context, inode Ino, owner uint64, ltype uint32, block bool) syscall.Errno
	// Getlk returns the current lock owner for a range on a file.
	Getlk(ctx Context, inode Ino, owner uint64, ltype *uint32, start, end *uint64, pid *uint32) syscall.Errno
	// Setlk sets a file range lock on given file.
	Setlk(ctx Context, inode Ino, owner uint64, block bool, ltype uint32, start, end uint64, pid uint32) syscall.Errno

	// Compact all the chunks by merge small slices together
	CompactAll(ctx Context, threads int, bar *utils.Bar) syscall.Errno
	// Compact chunks for specified path
	Compact(ctx Context, inode Ino, concurrency int, preFunc, postFunc func()) syscall.Errno

	// ListSlices returns all slices used by all files.
	ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno
	// Remove all files and directories recursively.
	// count represents the number of attempted deletions of entries (even if failed).
	Remove(ctx Context, parent Ino, name string, count *uint64) syscall.Errno
	// Get summary of a node; for a directory it will accumulate all its child nodes
	GetSummary(ctx Context, inode Ino, summary *Summary, recursive bool, strict bool) syscall.Errno
	// GetTreeSummary returns a summary in tree structure
	GetTreeSummary(ctx Context, root *TreeSummary, depth, topN uint8, strict bool, updateProgress func(count uint64, bytes uint64)) syscall.Errno
	// Clone a file or directory
	Clone(ctx Context, srcIno, dstParentIno Ino, dstName string, cmode uint8, cumask uint16, count, total *uint64) syscall.Errno
	// GetPaths returns all paths of an inode
	GetPaths(ctx Context, inode Ino) []string
	// Check integrity of an absolute path and repair it if asked
	Check(ctx Context, fpath string, repair bool, recursive bool, statAll bool) error
	// Change root to a directory specified by subdir
	Chroot(ctx Context, subdir string) syscall.Errno

	// Get a copy of the current format
	GetFormat() Format

	// OnMsg add a callback for the given message type.
	OnMsg(mtype uint32, cb MsgCallback)
	// OnReload register a callback for any change founded after reloaded.
	OnReload(func(new *Format))

	HandleQuota(ctx Context, cmd uint8, dpath string, quotas map[string]*Quota, strict, repair bool) error

	// Dump the tree under root, which may be modified by checkRoot
	DumpMeta(w io.Writer, root Ino, threads int, keepSecret, fast, skipTrash bool) error
	LoadMeta(r io.Reader) error

	InitMetrics(registerer prometheus.Registerer)

	SetFacl(ctx Context, ino Ino, aclType uint8, n *aclAPI.Rule) syscall.Errno
	GetFacl(ctx Context, ino Ino, aclType uint8, n *aclAPI.Rule) syscall.Errno
	// contains filtered or unexported methods
}

Meta is a interface for a meta service for file system.

func NewClient 

func NewClient(uri string, conf *Config) Meta

NewClient creates a Meta client for given uri.

type MsgCallback 

type MsgCallback func(...interface{}) error

MsgCallback is a callback for messages from meta service.

type PLockItem 

type PLockItem struct {
	// contains filtered or unexported fields
}

type Plock 

type Plock struct {
	Inode   Ino
	Owner   uint64
	Records []plockRecord
}

type Quota 

type Quota struct {
	MaxSpace, MaxInodes   int64
	UsedSpace, UsedInodes int64
	// contains filtered or unexported fields
}

type Session 

type Session struct {
	Sid    uint64
	Expire time.Time
	SessionInfo
	Sustained []Ino   `json:",omitempty"`
	Flocks    []Flock `json:",omitempty"`
	Plocks    []Plock `json:",omitempty"`
}

Session contains detailed information of a client session

type SessionInfo 

type SessionInfo struct {
	Version    string
	HostName   string
	IPAddrs    []string `json:",omitempty"`
	MountPoint string
	MountTime  time.Time
	ProcessID  int
}

type Slice 

type Slice struct {
	Id   uint64
	Size uint32
	Off  uint32
	Len  uint32
}

Slice is a slice of a chunk. Multiple slices could be combined together as a chunk.

type Summary 

type Summary struct {
	Length uint64
	Size   uint64
	Files  uint64
	Dirs   uint64
}

Summary represents the total number of files/directories and total length of all files inside a directory.

type TreeSummary 

type TreeSummary struct {
	Inode    Ino
	Path     string
	Type     uint8
	Size     uint64
	Files    uint64
	Dirs     uint64
	Children []*TreeSummary `json:",omitempty"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.