meta

package
v1.0.0-alpha5 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 6, 2023 License: Apache-2.0 Imports: 57 Imported by: 17

Documentation

Overview

nolint

Index

Constants

View Source 
const (
	// MaxVersion is the max of supported versions.
	MaxVersion = 1
	// ChunkSize is size of a chunk
	ChunkSize = 1 << 26 // 64M
	// DeleteSlice is a message to delete a slice from object store.
	DeleteSlice = 1000
	// CompactChunk is a message to compact a chunk in object store.
	CompactChunk = 1001
	// Rmr is a message to remove a directory recursively.
	Rmr = 1002
	// Info is a message to get the internal info for file or directory.
	Info = 1003
	// FillCache is a message to build cache for target directories/files
	FillCache = 1004
)
View Source 
const (
	TypeFile      = 1 // type for regular file
	TypeDirectory = 2 // type for directory
	TypeSymlink   = 3 // type for symlink
	TypeFIFO      = 4 // type for FIFO node
	TypeBlockDev  = 5 // type for block device
	TypeCharDev   = 6 // type for character device
	TypeSocket    = 7 // type for socket
)
View Source 
const (
	RenameNoReplace = 1 << iota
	RenameExchange
	RenameWhiteout
)
View Source 
const (
	// SetAttrMode is a mask to update a attribute of node
	SetAttrMode = 1 << iota
	SetAttrUID
	SetAttrGID
	SetAttrSize
	SetAttrAtime
	SetAttrMtime
	SetAttrCtime
	SetAttrAtimeNow
	SetAttrMtimeNow
)
View Source 
const (
	F_UNLCK = syscall.F_UNLCK
	F_RDLCK = syscall.F_RDLCK
	F_WRLCK = syscall.F_WRLCK
)
View Source 
const (
	XattrCreateOrReplace = 0
	XattrCreate          = sys.XATTR_CREATE
	XattrReplace         = sys.XATTR_REPLACE
)
View Source 
const CPROGRESS = 0xFE // 16 bytes: progress increment

Type of control messages

View Source 
const ENOATTR = syscall.ENODATA
View Source 
const MaxFieldsCountOfTable = 13 // node table
View Source 
const MaxName = 255
View Source 
const TrashName = ".trash"

Variables

View Source 
var CHARS = []byte("0123456789ABCDEF")

Functions

func GetPaths added in v1.0.0 

func GetPaths(m Meta, ctx Context, inode Ino) []string

Get all paths of an inode

func GetSummary added in v0.13.0 

func GetSummary(r Meta, ctx Context, inode Ino, summary *Summary, recursive bool) syscall.Errno

func Register 

func Register(name string, register Creator)

Types

type Attr 

type Attr struct {
	Flags     uint8  // reserved flags
	Typ       uint8  // type of a node
	Mode      uint16 // permission mode
	Uid       uint32 // owner id
	Gid       uint32 // group id of owner
	Rdev      uint32 // device number
	Atime     int64  // last access time
	Mtime     int64  // last modified time
	Ctime     int64  // last change time for meta
	Atimensec uint32 // nanosecond part of atime
	Mtimensec uint32 // nanosecond part of mtime
	Ctimensec uint32 // nanosecond part of ctime
	Nlink     uint32 // number of links (sub-directories or hardlinks)
	Length    uint64 // length of regular file

	Parent    Ino  // inode of parent; 0 means tracked by parentKey (for hardlinks)
	Full      bool // the attributes are completed or not
	KeepCache bool // whether to keep the cached page or not
}

Attr represents attributes of a node.

func (Attr) SMode 

func (a Attr) SMode() uint32

SMode is the file mode including type and unix permission.

type Config 

type Config struct {
	Strict      bool // update ctime
	Retries     int
	MaxDeletes  int
	CaseInsensi bool
	ReadOnly    bool
	NoBGJob     bool // disable background jobs like clean-up, backup, etc.
	OpenCache   time.Duration
	Heartbeat   time.Duration
	MountPoint  string
	Subdir      string
}

Config for clients.

type Context 

type Context interface {
	context.Context
	Gid() uint32
	Gids() []uint32
	Uid() uint32
	Pid() uint32
	WithValue(k, v interface{})
	Cancel()
	Canceled() bool
}
var Background Context = &emptyContext{context.Background()}

func NewContext 

func NewContext(pid, uid uint32, gids []uint32) Context

type Creator 

type Creator func(driver, addr string, conf *Config) (Meta, error)

type CtxKey 

type CtxKey string

type DumpedAttr 

type DumpedAttr struct {
	Inode     Ino    `json:"inode"`
	Type      string `json:"type"`
	Mode      uint16 `json:"mode"`
	Uid       uint32 `json:"uid"`
	Gid       uint32 `json:"gid"`
	Atime     int64  `json:"atime"`
	Mtime     int64  `json:"mtime"`
	Ctime     int64  `json:"ctime"`
	Atimensec uint32 `json:"atimensec,omitempty"`
	Mtimensec uint32 `json:"mtimensec,omitempty"`
	Ctimensec uint32 `json:"ctimensec,omitempty"`
	Nlink     uint32 `json:"nlink"`
	Length    uint64 `json:"length"`
	Rdev      uint32 `json:"rdev,omitempty"`
}

type DumpedChunk 

type DumpedChunk struct {
	Index  uint32         `json:"index"`
	Slices []*DumpedSlice `json:"slices"`
}

type DumpedCounters 

type DumpedCounters struct {
	UsedSpace         int64 `json:"usedSpace"`
	UsedInodes        int64 `json:"usedInodes"`
	NextInode         int64 `json:"nextInodes"`
	NextChunk         int64 `json:"nextChunk"`
	NextSession       int64 `json:"nextSession"`
	NextTrash         int64 `json:"nextTrash"`
	NextCleanupSlices int64 `json:"nextCleanupSlices,omitempty"` // deprecated, always 0
}

type DumpedDelFile 

type DumpedDelFile struct {
	Inode  Ino    `json:"inode"`
	Length uint64 `json:"length"`
	Expire int64  `json:"expire"`
}

type DumpedEntry 

type DumpedEntry struct {
	Name    string                  `json:"-"`
	Parents []Ino                   `json:"-"`
	Attr    *DumpedAttr             `json:"attr,omitempty"`
	Symlink string                  `json:"symlink,omitempty"`
	Xattrs  []*DumpedXattr          `json:"xattrs,omitempty"`
	Chunks  []*DumpedChunk          `json:"chunks,omitempty"`
	Entries map[string]*DumpedEntry `json:"entries,omitempty"`
}

type DumpedMeta 

type DumpedMeta struct {
	Setting   Format
	Counters  *DumpedCounters
	Sustained []*DumpedSustained
	DelFiles  []*DumpedDelFile
	FSTree    *DumpedEntry `json:",omitempty"`
	Trash     *DumpedEntry `json:",omitempty"`
}

type DumpedSlice 

type DumpedSlice struct {
	Chunkid uint64 `json:"chunkid,omitempty"`
	Id      uint64 `json:"id"`
	Pos     uint32 `json:"pos,omitempty"`
	Size    uint32 `json:"size"`
	Off     uint32 `json:"off,omitempty"`
	Len     uint32 `json:"len"`
}

type DumpedSustained 

type DumpedSustained struct {
	Sid    uint64 `json:"sid"`
	Inodes []Ino  `json:"inodes"`
}

type DumpedXattr 

type DumpedXattr struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

type Entry 

type Entry struct {
	Inode Ino
	Name  []byte
	Attr  *Attr
}

Entry is an entry inside a directory.

type Flock 

type Flock struct {
	Inode Ino
	Owner uint64
	Ltype string
}

type Format 

type Format struct {
	Name             string
	UUID             string
	Storage          string
	Bucket           string
	AccessKey        string `json:",omitempty"`
	SecretKey        string `json:",omitempty"`
	SessionToken     string `json:",omitempty"`
	BlockSize        int
	Compression      string `json:",omitempty"`
	Shards           int    `json:",omitempty"`
	HashPrefix       bool   `json:",omitempty"`
	Capacity         uint64 `json:",omitempty"`
	Inodes           uint64 `json:",omitempty"`
	EncryptKey       string `json:",omitempty"`
	KeyEncrypted     bool   `json:",omitempty"`
	TrashDays        int    `json:",omitempty"`
	MetaVersion      int    `json:",omitempty"`
	MinClientVersion string `json:",omitempty"`
	MaxClientVersion string `json:",omitempty"`
}

func (*Format) CheckVersion 

func (f *Format) CheckVersion() error

func (*Format) Decrypt 

func (f *Format) Decrypt() error

func (*Format) Encrypt 

func (f *Format) Encrypt() error

func (*Format) RemoveSecret 

func (f *Format) RemoveSecret()

func (*Format) String 

func (f *Format) String() string

type Ino 

type Ino uint64
const RootInode Ino = 1
const TrashInode Ino = 0x7FFFFFFF10000000 // larger than vfs.minInternalNode

func (Ino) String 

func (i Ino) String() string

type Meta 

type Meta interface {
	// Name of database
	Name() string
	// Init is used to initialize a meta service.
	Init(format Format, force bool) error
	// Shutdown close current database connections.
	Shutdown() error
	// Reset cleans up all metadata, VERY DANGEROUS!
	Reset() error
	// Load loads the existing setting of a formatted volume from meta service.
	Load(checkVersion bool) (*Format, error)
	// NewSession creates a new client session.
	NewSession() error
	// CloseSession does cleanup and close the session.
	CloseSession() error
	// GetSession retrieves information of session with sid
	GetSession(sid uint64, detail bool) (*Session, error)
	// ListSessions returns all client sessions.
	ListSessions() ([]*Session, error)
	// CleanStaleSessions cleans up sessions not active for more than 5 minutes
	CleanStaleSessions()

	// StatFS returns summary statistics of a volume.
	StatFS(ctx Context, totalspace, availspace, iused, iavail *uint64) syscall.Errno
	// Access checks the access permission on given inode.
	Access(ctx Context, inode Ino, modemask uint8, attr *Attr) syscall.Errno
	// Lookup returns the inode and attributes for the given entry in a directory.
	Lookup(ctx Context, parent Ino, name string, inode *Ino, attr *Attr) syscall.Errno
	// Resolve fetches the inode and attributes for an entry identified by the given path.
	// ENOTSUP will be returned if there's no natural implementation for this operation or
	// if there are any symlink following involved.
	Resolve(ctx Context, parent Ino, path string, inode *Ino, attr *Attr) syscall.Errno
	// GetAttr returns the attributes for given node.
	GetAttr(ctx Context, inode Ino, attr *Attr) syscall.Errno
	// SetAttr updates the attributes for given node.
	SetAttr(ctx Context, inode Ino, set uint16, sggidclearmode uint8, attr *Attr) syscall.Errno
	// Truncate changes the length for given file.
	Truncate(ctx Context, inode Ino, flags uint8, attrlength uint64, attr *Attr) syscall.Errno
	// Fallocate preallocate given space for given file.
	Fallocate(ctx Context, inode Ino, mode uint8, off uint64, size uint64) syscall.Errno
	// ReadLink returns the target of a symlink.
	ReadLink(ctx Context, inode Ino, path *[]byte) syscall.Errno
	// Symlink creates a symlink in a directory with given name.
	Symlink(ctx Context, parent Ino, name string, path string, inode *Ino, attr *Attr) syscall.Errno
	// Mknod creates a node in a directory with given name, type and permissions.
	Mknod(ctx Context, parent Ino, name string, _type uint8, mode uint16, cumask uint16, rdev uint32, path string, inode *Ino, attr *Attr) syscall.Errno
	// Mkdir creates a sub-directory with given name and mode.
	Mkdir(ctx Context, parent Ino, name string, mode uint16, cumask uint16, copysgid uint8, inode *Ino, attr *Attr) syscall.Errno
	// Unlink removes a file entry from a directory.
	// The file will be deleted if it's not linked by any entries and not open by any sessions.
	Unlink(ctx Context, parent Ino, name string) syscall.Errno
	// Rmdir removes an empty sub-directory.
	Rmdir(ctx Context, parent Ino, name string) syscall.Errno
	// Rename move an entry from a source directory to another with given name.
	// The targeted entry will be overwrited if it's a file or empty directory.
	// For Hadoop, the target should not be overwritten.
	Rename(ctx Context, parentSrc Ino, nameSrc string, parentDst Ino, nameDst string, flags uint32, inode *Ino, attr *Attr) syscall.Errno
	// Link creates an entry for node.
	Link(ctx Context, inodeSrc, parent Ino, name string, attr *Attr) syscall.Errno
	// Readdir returns all entries for given directory, which include attributes if plus is true.
	Readdir(ctx Context, inode Ino, wantattr uint8, entries *[]*Entry) syscall.Errno
	// Create creates a file in a directory with given name.
	Create(ctx Context, parent Ino, name string, mode uint16, cumask uint16, flags uint32, inode *Ino, attr *Attr) syscall.Errno
	// Open checks permission on a node and track it as open.
	Open(ctx Context, inode Ino, flags uint32, attr *Attr) syscall.Errno
	// Close a file.
	Close(ctx Context, inode Ino) syscall.Errno
	// Read returns the list of slices on the given chunk.
	Read(ctx Context, inode Ino, indx uint32, slices *[]Slice) syscall.Errno
	// NewSlice returns an id for new slice.
	NewSlice(ctx Context, id *uint64) syscall.Errno
	// Write put a slice of data on top of the given chunk.
	Write(ctx Context, inode Ino, indx uint32, off uint32, slice Slice) syscall.Errno
	// InvalidateChunkCache invalidate chunk cache
	InvalidateChunkCache(ctx Context, inode Ino, indx uint32) syscall.Errno
	// CopyFileRange copies part of a file to another one.
	CopyFileRange(ctx Context, fin Ino, offIn uint64, fout Ino, offOut uint64, size uint64, flags uint32, copied *uint64) syscall.Errno
	// GetParents returns a map of node parents (> 1 parents if hardlinked)
	GetParents(ctx Context, inode Ino) map[Ino]int

	// GetXattr returns the value of extended attribute for given name.
	GetXattr(ctx Context, inode Ino, name string, vbuff *[]byte) syscall.Errno
	// ListXattr returns all extended attributes of a node.
	ListXattr(ctx Context, inode Ino, dbuff *[]byte) syscall.Errno
	// SetXattr update the extended attribute of a node.
	SetXattr(ctx Context, inode Ino, name string, value []byte, flags uint32) syscall.Errno
	// RemoveXattr removes the extended attribute of a node.
	RemoveXattr(ctx Context, inode Ino, name string) syscall.Errno
	// Flock tries to put a lock on given file.
	Flock(ctx Context, inode Ino, owner uint64, ltype uint32, block bool) syscall.Errno
	// Getlk returns the current lock owner for a range on a file.
	Getlk(ctx Context, inode Ino, owner uint64, ltype *uint32, start, end *uint64, pid *uint32) syscall.Errno
	// Setlk sets a file range lock on given file.
	Setlk(ctx Context, inode Ino, owner uint64, block bool, ltype uint32, start, end uint64, pid uint32) syscall.Errno

	// Compact all the chunks by merge small slices together
	CompactAll(ctx Context, bar *utils.Bar) syscall.Errno
	// ListSlices returns all slices used by all files.
	ListSlices(ctx Context, slices map[Ino][]Slice, delete bool, showProgress func()) syscall.Errno
	// Remove all files and directories recursively.
	Remove(ctx Context, parent Ino, name string, count *uint64) syscall.Errno

	// OnMsg add a callback for the given message type.
	OnMsg(mtype uint32, cb MsgCallback)

	// Dump the tree under root, which may be modified by checkRoot
	DumpMeta(w io.Writer, root Ino) error
	LoadMeta(r io.Reader) error

	InitMetrics(registerer prometheus.Registerer)
	// contains filtered or unexported methods
}

Meta is a interface for a meta service for file system.

func NewClient 

func NewClient(uri string, conf *Config) Meta

NewClient creates a Meta client for given uri.

type MsgCallback 

type MsgCallback func(...interface{}) error

MsgCallback is a callback for messages from meta service.

type Plock 

type Plock struct {
	Inode   Ino
	Owner   uint64
	Records []byte // FIXME: loadLocks
}

type Session 

type Session struct {
	Sid    uint64
	Expire time.Time
	SessionInfo
	Sustained []Ino   `json:",omitempty"`
	Flocks    []Flock `json:",omitempty"`
	Plocks    []Plock `json:",omitempty"`
}

Session contains detailed information of a client session

type SessionInfo 

type SessionInfo struct {
	Version    string
	HostName   string
	MountPoint string
	ProcessID  int
}

type Slice 

type Slice struct {
	Id   uint64
	Size uint32
	Off  uint32
	Len  uint32
}

Slice is a slice of a chunk. Multiple slices could be combined together as a chunk.

type Summary 

type Summary struct {
	Length uint64
	Size   uint64
	Files  uint64
	Dirs   uint64
}

Summary represents the total number of files/directories and total length of all files inside a directory.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.