cluster

package
v1.3.16 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 31, 2023 License: MIT Imports: 32 Imported by: 0

Documentation

Overview

Package cluster provides local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package cluster provides local access to cluster-level metadata

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package cluster provides common interfaces and local access to cluster-level metadata

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package cluster provides local access to cluster-level metadata

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package cluster provides local access to cluster-level metadata

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source
const (
	XattrLOM = "user.ais.lom" // on-disk xattr name

	DumpLomEnvVar = "AIS_DUMP_LOM"
)
View Source
const (
	Targets = iota // 0 (cluster.Targets) used as default value for NewStreamBundle
	Proxies
	AllNodes
)
View Source
const (
	SnodeNonElectable cos.BitFlags = 1 << iota
	SnodeIC
	NodeFlagMaint
	NodeFlagDecomm
)

enum Snode.Flags

View Source
const (
	PnamePrefix = "p["
	TnamePrefix = "t["
	SnameSuffix = "]"
)
View Source
const (
	UponTerm     = Upon(1 << iota) // success or fail is separately provided via error
	UponProgress                   // periodic (BytesCount, ObjCount)
)

enum: when to notify

View Source
const (
	QuiInactiveCB = QuiRes(iota) // e.g., no pending requests (NOTE: used exclusively by `quicb` callbacks)
	QuiActive                    // active (e.g., receiving data)
	QuiActiveRet                 // active that immediately breaks waiting for quiecscence
	QuiDone                      // all done
	QuiAborted                   // aborted
	QuiTimeout                   // timeout
	Quiescent                    // idle => quiescent
)
View Source
const MaxBcastMultiplier = 2

number of broadcasting goroutines <= cmn.NumCPU() * MaxBcastMultiplier

View Source
const NodeFlagsMaintDecomm = NodeFlagMaint | NodeFlagDecomm

Variables

This section is empty.

Functions

func EvictLomCache

func EvictLomCache(b *Bck)

func FreeCpObjParams

func FreeCpObjParams(a *CopyObjectParams)

func FreeLOM

func FreeLOM(lom *LOM)

func FreeNodes

func FreeNodes(nodes Nodes)

func FreePutObjParams

func FreePutObjParams(a *PutObjectParams)

func HrwFQN

func HrwFQN(bck *cmn.Bck, contentType, objName string) (fqn string, digest uint64, err error)

func HrwMpath

func HrwMpath(uname string) (mi *fs.Mountpath, digest uint64, err error)

func Init

func Init(t TargetLoc)

func MaxBcastParallel

func MaxBcastParallel() int

func N2ID

func N2ID(name string) string

func ParseObjLoc

func ParseObjLoc(loc string) (tname, mpname string)

func Pname

func Pname(pid string) string

func RegLomCacheWithHK

func RegLomCacheWithHK(t Target)

func ResolveFQN

func ResolveFQN(fqn string) (parsedFQN fs.ParsedFQN, hrwFQN string, err error)

func Tname

func Tname(tid string) string

Types

type BMD

type BMD struct {
	Ext       any       `json:"ext,omitempty"`  // within meta-version extensions
	Providers Providers `json:"providers"`      // (provider, namespace, bucket) hierarchy
	UUID      string    `json:"uuid"`           // unique & immutable
	Version   int64     `json:"version,string"` // gets incremented on every update
}

- BMD is the root of the (providers, namespaces, buckets) hierarchy - BMD (instance) can be obtained via Bowner.Get() - BMD is immutable and versioned - BMD versioning is monotonic and incremental

func (*BMD) Add

func (m *BMD) Add(bck *Bck)

func (*BMD) Del

func (m *BMD) Del(bck *Bck) (deleted bool)

func (*BMD) Exists

func (m *BMD) Exists(bck *Bck, bckID uint64) (exists bool)

func (*BMD) Get

func (m *BMD) Get(bck *Bck) (p *cmn.BucketProps, present bool)

func (*BMD) IsECUsed

func (m *BMD) IsECUsed() (yes bool)

func (*BMD) JspOpts

func (*BMD) JspOpts() jsp.Options

func (*BMD) Range

func (m *BMD) Range(providerQuery *string, nsQuery *cmn.Ns, callback func(*Bck) bool)

providerQuery == nil: all providers; nsQuery == nil: all namespaces

func (*BMD) Select

func (m *BMD) Select(qbck *cmn.QueryBcks) cmn.Bcks

func (*BMD) Set

func (m *BMD) Set(bck *Bck, p *cmn.BucketProps)

func (*BMD) String

func (m *BMD) String() string

func (*BMD) StringEx

func (m *BMD) StringEx() string

type BackendProvider

type BackendProvider interface {
	Provider() string
	MaxPageSize() uint
	CreateBucket(bck *Bck) (errCode int, err error)
	ListObjects(bck *Bck, msg *apc.LsoMsg, lst *cmn.LsoResult) (errCode int, err error)
	ListBuckets(qbck cmn.QueryBcks) (bcks cmn.Bcks, errCode int, err error)
	PutObj(r io.ReadCloser, lom *LOM) (errCode int, err error)
	DeleteObj(lom *LOM) (errCode int, err error)

	// with context
	HeadBucket(ctx context.Context, bck *Bck) (bckProps cos.StrKVs, errCode int, err error)
	HeadObj(ctx context.Context, lom *LOM) (objAttrs *cmn.ObjAttrs, errCode int, err error)
	GetObj(ctx context.Context, lom *LOM, owt cmn.OWT) (errCode int, err error)
	GetObjReader(ctx context.Context, lom *LOM) (r io.ReadCloser, expectedCksum *cos.Cksum, errCode int, err error)
}

type Bck

type Bck cmn.Bck

func CloneBck

func CloneBck(bck *cmn.Bck) *Bck

func InitByNameOnly

func InitByNameOnly(bckName string, bowner Bowner) (bck *Bck, err error, errCode int)

to support s3 clients: find an already existing bucket by name (and nothing else) returns an error when name cannot be unambiguously resolved to a single bucket

func NewBck

func NewBck(name, provider string, ns cmn.Ns, optProps ...*cmn.BucketProps) *Bck

func (*Bck) AddToQuery

func (b *Bck) AddToQuery(q url.Values) url.Values

func (*Bck) AddUnameToQuery

func (b *Bck) AddUnameToQuery(q url.Values, uparam string) url.Values

func (*Bck) Allow

func (b *Bck) Allow(bit apc.AccessAttrs) error

func (*Bck) Backend

func (b *Bck) Backend() *Bck

func (*Bck) Bucket

func (b *Bck) Bucket() *cmn.Bck

cast *cluster.Bck => *cmn.Bck

func (*Bck) CksumConf

func (b *Bck) CksumConf() (conf *cmn.CksumConf)

func (*Bck) Clone

func (b *Bck) Clone() cmn.Bck

clone (*cluster.Bck | *cmn.Bck) <=> (cmn.Bck | cluster.Bck) respectively

func (*Bck) Cname added in v1.3.16

func (b *Bck) Cname(name string) string

func (*Bck) Equal

func (b *Bck) Equal(other *Bck, sameID, sameBackend bool) bool

func (*Bck) GetNameLockPair

func (b *Bck) GetNameLockPair() (nlp *NameLockPair)

func (*Bck) HasProvider

func (b *Bck) HasProvider() bool

func (*Bck) Init

func (b *Bck) Init(bowner Bowner) (err error)

when the bucket is not present in the BMD: - always returns the corresponding *DoesNotExist error - Cloud bucket: fills in the props with defaults from config - AIS bucket: sets the props to nil - Remote (Cloud or Remote AIS) bucket: caller can type-cast err.(*cmn.ErrRemoteBckNotFound) and proceed

func (*Bck) InitNoBackend

func (b *Bck) InitNoBackend(bowner Bowner) error

func (*Bck) IsAIS

func (b *Bck) IsAIS() bool

func (*Bck) IsCloud

func (b *Bck) IsCloud() bool

func (*Bck) IsEmpty

func (b *Bck) IsEmpty() bool

func (*Bck) IsHDFS

func (b *Bck) IsHDFS() bool

func (*Bck) IsHTTP

func (b *Bck) IsHTTP() bool

func (*Bck) IsQuery

func (b *Bck) IsQuery() bool

func (*Bck) IsRemote

func (b *Bck) IsRemote() bool

func (*Bck) IsRemoteAIS

func (b *Bck) IsRemoteAIS() bool

func (*Bck) MakeUname

func (b *Bck) MakeUname(name string) string

func (*Bck) MaskBID

func (b *Bck) MaskBID(i int64) uint64

func (*Bck) RemoteBck

func (b *Bck) RemoteBck() *cmn.Bck

func (*Bck) String

func (b *Bck) String() string

func (*Bck) Validate

func (b *Bck) Validate() error

func (*Bck) VersionConf

func (b *Bck) VersionConf() cmn.VersionConf

type Bowner

type Bowner interface {
	Get() (bmd *BMD)
}

interface to Get current BMD instance (for implementation, see ais/bucketmeta.go)

type Buckets

type Buckets map[string]*cmn.BucketProps

interface to Get current BMD instance (for implementation, see ais/bucketmeta.go)

type CT

type CT struct {
	// contains filtered or unexported fields
}

func NewCTFromBO

func NewCTFromBO(bck *cmn.Bck, objName string, b Bowner, ctType ...string) (ct *CT, err error)

func NewCTFromFQN

func NewCTFromFQN(fqn string, b Bowner) (ct *CT, err error)

func NewCTFromLOM

func NewCTFromLOM(lom *LOM, ctType string) *CT

Construct CT from LOM and change ContentType and FQN

func (*CT) Bck

func (ct *CT) Bck() *Bck

func (*CT) Bucket

func (ct *CT) Bucket() *cmn.Bck

func (*CT) CacheIdx

func (ct *CT) CacheIdx() int

func (*CT) Clone

func (ct *CT) Clone(ctType string) *CT

Clone CT and change ContentType and FQN

func (*CT) ContentType

func (ct *CT) ContentType() string

func (*CT) FQN

func (ct *CT) FQN() string

func (*CT) LoadFromFS

func (ct *CT) LoadFromFS() error

func (*CT) Lock

func (ct *CT) Lock(exclusive bool)

func (*CT) Make

func (ct *CT) Make(toType string, pref ...string) string

func (*CT) Mountpath added in v1.3.16

func (ct *CT) Mountpath() *fs.Mountpath

func (*CT) MtimeUnix

func (ct *CT) MtimeUnix() int64

func (*CT) ObjectName

func (ct *CT) ObjectName() string

func (*CT) SizeBytes

func (ct *CT) SizeBytes() int64

func (*CT) Uname

func (ct *CT) Uname() string

func (*CT) Unlock

func (ct *CT) Unlock(exclusive bool)

func (*CT) Write

func (ct *CT) Write(t Target, reader io.Reader, size int64, workFQN ...string) (err error)

Save CT to local drives. If workFQN is set, it saves in two steps: first, save to workFQN; second, rename workFQN to ct.FQN. If unset, it writes directly to ct.FQN

type CopyObjectParams

type CopyObjectParams struct {
	DM        DataMover
	DP        DP // Data Provider (optional; see Transform/Copy Bucket (TCB))
	Xact      Xact
	BckTo     *Bck
	ObjNameTo string
	Buf       []byte
}

data path: control structures and types

func AllocCpObjParams

func AllocCpObjParams() (a *CopyObjectParams)

type DP

type DP interface {
	Reader(lom *LOM) (reader cos.ReadOpenCloser, oah cmn.ObjAttrsHolder, err error)
}

data provider

type DataMover

type DataMover interface {
	RegRecv() error
	GetXact() Xact
	Open()
	Close(err error)
	UnregRecv()
	Send(obj *transport.Obj, roc cos.ReadOpenCloser, tsi *Snode) error
	ACK(hdr transport.ObjHdr, cb transport.ObjSentCB, tsi *Snode) error
	OWT() cmn.OWT
}

data path: control structures and types

type LDP

type LDP struct{}

func (*LDP) Reader

func (*LDP) Reader(lom *LOM) (cos.ReadOpenCloser, cmn.ObjAttrsHolder, error)

compare with etl/dp.go

type LIF

type LIF struct {
	Uname string
	BID   uint64
	// contains filtered or unexported fields
}

LOM In Flight (LIF)

func (*LIF) CacheIdx

func (lif *LIF) CacheIdx() int

deferred unlocking

func (*LIF) LOM

func (lif *LIF) LOM() (lom *LOM, err error)

LIF => LOF with a check for bucket existence

func (*LIF) Unlock

func (lif *LIF) Unlock(exclusive bool)

type LOM

type LOM struct {
	ObjName string

	FQN    string
	HrwFQN string // (=> main replica)
	// contains filtered or unexported fields
}

func AllocLOM

func AllocLOM(objName string) *LOM

func AllocLomFromHdr

func AllocLomFromHdr(hdr *transport.ObjHdr) (lom *LOM, err error)

lom <= transport.ObjHdr (NOTE: caller must call freeLOM)

func (*LOM) AcquireAtimefs

func (lom *LOM) AcquireAtimefs() error

used in tests

func (*LOM) AddCopy

func (lom *LOM) AddCopy(copyFQN string, mpi *fs.Mountpath) error

NOTE: used only in tests

func (*LOM) AllowDisconnectedBackend

func (lom *LOM) AllowDisconnectedBackend(loaded bool) (err error)

permission to overwrite objects that were previously read from: a) any remote backend that is currently not configured as the bucket's backend b) HTPP ("ht://") since it's not writable

func (*LOM) Atime

func (lom *LOM) Atime() time.Time

func (*LOM) AtimeUnix

func (lom *LOM) AtimeUnix() int64

func (*LOM) Bck

func (lom *LOM) Bck() *Bck

func (*LOM) Bprops

func (lom *LOM) Bprops() *cmn.BucketProps

func (*LOM) Bucket

func (lom *LOM) Bucket() *cmn.Bck

func (*LOM) CacheIdx

func (lom *LOM) CacheIdx() int

func (*LOM) Checksum

func (lom *LOM) Checksum() *cos.Cksum

func (*LOM) CksumConf

func (lom *LOM) CksumConf() *cmn.CksumConf

func (*LOM) CksumType

func (lom *LOM) CksumType() string

func (*LOM) CloneMD

func (lom *LOM) CloneMD(fqn string) *LOM

allocates and copies metadata (in particular, atime and uname)

func (*LOM) Cname added in v1.3.16

func (lom *LOM) Cname() string

see also: transport.ObjHdr.Cname()

func (*LOM) ComputeCksum

func (lom *LOM) ComputeCksum(cksumType string) (cksum *cos.CksumHash, err error)

func (*LOM) ComputeSetCksum

func (lom *LOM) ComputeSetCksum() (*cos.Cksum, error)

func (*LOM) Copy

func (lom *LOM) Copy(mi *fs.Mountpath, buf []byte) (err error)

increment the object's num copies by (well) copying the former (compare with lom.Copy2FQN below)

func (*LOM) Copy2FQN

func (lom *LOM) Copy2FQN(dstFQN string, buf []byte) (dst *LOM, err error)

copy object => any local destination recommended for copying between different buckets (compare with lom.Copy() above) NOTE: `lom` source must be w-locked

func (*LOM) CopyAttrs

func (lom *LOM) CopyAttrs(oah cmn.ObjAttrsHolder, skipCksum bool)

func (*LOM) CreateFile

func (lom *LOM) CreateFile(fqn string) (fh *os.File, err error)

(compare with cos.CreateFile)

func (*LOM) DelAllCopies

func (lom *LOM) DelAllCopies() (err error)

func (*LOM) DelCopies

func (lom *LOM) DelCopies(copiesFQN ...string) (err error)

func (*LOM) DelExtraCopies

func (lom *LOM) DelExtraCopies(fqn ...string) (removed bool, err error)

DelExtraCopies deletes obj replicas that are not part of the lom.md.copies metadata (cleanup)

func (*LOM) DowngradeLock

func (lom *LOM) DowngradeLock()

func (*LOM) ECEnabled

func (lom *LOM) ECEnabled() bool

func (*LOM) EqCksum

func (lom *LOM) EqCksum(cksum *cos.Cksum) bool

func (*LOM) Equal

func (lom *LOM) Equal(rem cmn.ObjAttrsHolder) (equal bool)

LOM == remote-object equality check

func (*LOM) FromFS

func (lom *LOM) FromFS() error

func (*LOM) GetCopies

func (lom *LOM) GetCopies() fs.MPI

GetCopies returns all copies (NOTE that copies include self) NOTE: caller must take a lock

func (*LOM) GetCustomKey

func (lom *LOM) GetCustomKey(key string) (string, bool)

func (*LOM) GetCustomMD

func (lom *LOM) GetCustomMD() cos.StrKVs

custom metadata

func (*LOM) HasCopies

func (lom *LOM) HasCopies() bool

func (*LOM) HrwTarget

func (lom *LOM) HrwTarget(smap *Smap) (tsi *Snode, local bool, err error)

func (*LOM) IncVersion

func (lom *LOM) IncVersion() error

func (*LOM) InitBck

func (lom *LOM) InitBck(bck *cmn.Bck) (err error)

func (*LOM) InitCT

func (lom *LOM) InitCT(ct *CT)

func (*LOM) InitFQN

func (lom *LOM) InitFQN(fqn string, expbck *cmn.Bck) (err error)

func (*LOM) IsCopy

func (lom *LOM) IsCopy() bool

given an existing (on-disk) object, determines whether it is a _copy_ (compare with isMirror below)

func (*LOM) IsHRW

func (lom *LOM) IsHRW() bool

func (*LOM) IsLocked

func (lom *LOM) IsLocked() (int, bool)

func (*LOM) LBGet

func (lom *LOM) LBGet() (fqn string)

load-balanced GET

func (*LOM) LIF

func (lom *LOM) LIF() (lif LIF)

constructor

func (*LOM) LeastUtilNoCopy

func (lom *LOM) LeastUtilNoCopy() (mi *fs.Mountpath)

returns the least utilized mountpath that does _not_ have a copy of this `lom` yet (compare with leastUtilCopy())

func (*LOM) Load

func (lom *LOM) Load(cacheit, locked bool) (err error)
  • locked: is locked by the immediate caller (or otherwise is known to be locked); if false, try Rlock temporarily *if and only when* reading from FS

func (*LOM) LoadMetaFromFS

func (lom *LOM) LoadMetaFromFS() error

NOTE: used in tests, ignores `dirty`

func (*LOM) Location

func (lom *LOM) Location() string

func (*LOM) Lock

func (lom *LOM) Lock(exclusive bool)

func (*LOM) MirrorConf

func (lom *LOM) MirrorConf() *cmn.MirrorConf

func (*LOM) Mountpath added in v1.3.16

func (lom *LOM) Mountpath() *fs.Mountpath

func (*LOM) NewDeferROC

func (lom *LOM) NewDeferROC() (cos.ReadOpenCloser, error)

is called under rlock; unlocks on fail

func (*LOM) NumCopies

func (lom *LOM) NumCopies() int

func (*LOM) ObjAttrs

func (lom *LOM) ObjAttrs() *cmn.ObjAttrs

func (*LOM) ObjectName

func (lom *LOM) ObjectName() string

as fs.PartsFQN

func (*LOM) Persist

func (lom *LOM) Persist() (err error)

(caller must set atime; compare with the above)

func (*LOM) PersistMain

func (lom *LOM) PersistMain() (err error)

(copy/paste tradeoff on purpose)

func (*LOM) Recache

func (lom *LOM) Recache()

store new or refresh existing

func (*LOM) Remove

func (lom *LOM) Remove(force ...bool) (err error)

func (*LOM) RenameFile

func (lom *LOM) RenameFile(workfqn string) error

(compare with cos.Rename)

func (*LOM) RestoreToLocation

func (lom *LOM) RestoreToLocation() (exists bool)

RestoreObjectFromAny tries to restore the object at its default location. Returns true if object exists, false otherwise TODO: locking vs concurrent restore: consider (read-lock object + write-lock meta) split

func (*LOM) SetAtimeUnix

func (lom *LOM) SetAtimeUnix(tu int64)

func (*LOM) SetCksum

func (lom *LOM) SetCksum(cksum *cos.Cksum)

func (*LOM) SetCustomKey

func (lom *LOM) SetCustomKey(key, value string)

func (*LOM) SetCustomMD

func (lom *LOM) SetCustomMD(md cos.StrKVs)

func (*LOM) SetSize

func (lom *LOM) SetSize(size int64)

func (*LOM) SetVersion

func (lom *LOM) SetVersion(ver string)

func (*LOM) SizeBytes

func (lom *LOM) SizeBytes(special ...bool) int64

special a) when a new version is being created b) for usage in unit tests

func (*LOM) String

func (lom *LOM) String() string

func (*LOM) StringEx

func (lom *LOM) StringEx() string

func (*LOM) ToMpath

func (lom *LOM) ToMpath() (mi *fs.Mountpath, isHrw bool)

must be called under w-lock returns mountpath destination to copy this object, or nil if no copying is required - checks hrw location first, and - checks copies (if any) against the current configuation and available mountpaths; - does not check `fstat` in either case (TODO: configurable or scrub);

func (*LOM) TryLock

func (lom *LOM) TryLock(exclusive bool) bool

func (*LOM) Uname

func (lom *LOM) Uname() string

func (*LOM) Uncache

func (lom *LOM) Uncache(delDirty bool)

func (*LOM) Unlock

func (lom *LOM) Unlock(exclusive bool)

func (*LOM) UpgradeLock

func (lom *LOM) UpgradeLock() (finished bool)

func (*LOM) ValidateContentChecksum

func (lom *LOM) ValidateContentChecksum() (err error)

ValidateDiskChecksum validates if checksum stored in lom's in-memory metadata matches object's content checksum. Use lom.ValidateMetaChecksum() to check lom's checksum vs on-disk metadata.

func (*LOM) ValidateMetaChecksum

func (lom *LOM) ValidateMetaChecksum() error

ValidateMetaChecksum validates whether checksum stored in lom's in-memory metadata matches checksum stored on disk. Use lom.ValidateContentChecksum() to recompute and check object's content checksum.

func (*LOM) Version

func (lom *LOM) Version(special ...bool) string

func (*LOM) VersionConf

func (lom *LOM) VersionConf() cmn.VersionConf

func (*LOM) WritePolicy

func (lom *LOM) WritePolicy() (p apc.WritePolicy)

type NameLockPair

type NameLockPair struct {
	// contains filtered or unexported fields
}

func (*NameLockPair) Lock

func (nlp *NameLockPair) Lock()

func (*NameLockPair) TryLock

func (nlp *NameLockPair) TryLock(timeout time.Duration) (ok bool)

func (*NameLockPair) TryRLock

func (nlp *NameLockPair) TryRLock(timeout time.Duration) (ok bool)

TODO: ensure single-time usage (no ref counting!)

func (*NameLockPair) Unlock

func (nlp *NameLockPair) Unlock()

type Namespaces

type Namespaces map[string]Buckets

interface to Get current BMD instance (for implementation, see ais/bucketmeta.go)

type NetInfo

type NetInfo struct {
	Hostname string `json:"node_ip_addr"`
	Port     string `json:"daemon_port"`
	URL      string `json:"direct_url"`
	// contains filtered or unexported fields
}

Snode's networking info

func NewNetInfo

func NewNetInfo(proto, hostname, port string) *NetInfo

func (*NetInfo) String

func (ni *NetInfo) String() string

func (*NetInfo) TCPEndpoint

func (ni *NetInfo) TCPEndpoint() string

type Node

type Node interface {
	SID() string
	String() string
	Snode() *Snode

	Bowner() Bowner
	Sowner() Sowner

	ClusterStarted() bool
	NodeStarted() bool
}

type NodeMap

type NodeMap map[string]*Snode // map of Snodes: DaeID => Snodes

func NodeMapDelta

func NodeMapDelta(oldNodeMap, newNodeMap []NodeMap) (added, removed NodeMap)

helper to find out NodeMap "delta" or "diff"

func (NodeMap) ActiveMap

func (m NodeMap) ActiveMap() (clone NodeMap)

func (NodeMap) ActiveNodes

func (m NodeMap) ActiveNodes() []*Snode

func (NodeMap) Add

func (m NodeMap) Add(snode *Snode)

func (NodeMap) Contains

func (m NodeMap) Contains(daeID string) (exists bool)

type NodeMemCap added in v1.3.16

type NodeMemCap interface {
	Node

	// Memory allocators
	PageMM() *memsys.MMSA
	ByteMM() *memsys.MMSA

	// Space
	OOS(*fs.CapStatus) fs.CapStatus

	// Running now
	GetAllRunning(xactKind string, separateIdle bool) (running, idle []string)
}

type Nodes

type Nodes []*Snode // slice of Snodes

func AllocNodes

func AllocNodes(capacity int) (nodes Nodes)

func HrwTargetList

func HrwTargetList(uname string, smap *Smap, count int) (sis Nodes, err error)

Sorts all targets in a cluster by their respective HRW (weights) in a descending order; returns resulting subset (aka slice) that has the requested length = count. Returns error if the cluster does not have enough targets. If count == length of Smap.Tmap, the function returns as many targets as possible.

type Notif

type Notif interface {
	OnFinishedCB() func(Notif, error)
	OnProgressCB() func(Notif)
	NotifyInterval() time.Duration // notify interval in secs
	LastNotifTime() int64          // time last notified
	SetLastNotified(now int64)
	Upon(u Upon) bool
	Subscribers() []string
	ToNotifMsg() NotifMsg
}

intra-cluster notification interface

type NotifMsg

type NotifMsg struct {
	UUID   string `json:"uuid"`    // xaction UUID
	NodeID string `json:"node_id"` // notifier node ID
	Kind   string `json:"kind"`    // xaction `Kind`
	ErrMsg string `json:"err"`     // error.Error()
	Data   []byte `json:"message"` // typed message
}

func (*NotifMsg) String

func (msg *NotifMsg) String() (s string)

type OnFinishObj

type OnFinishObj = func(lom *LOM, err error)

data path: control structures and types

type PromoteArgs

type PromoteArgs struct {
	DaemonID  string `json:"tid,omitempty"` // target ID
	SrcFQN    string `json:"src,omitempty"` // source file or directory (must be absolute pathname)
	ObjName   string `json:"obj,omitempty"` // destination object name
	Recursive bool   `json:"rcr,omitempty"` // recursively promote nested dirs
	// once successfully promoted:
	OverwriteDst bool `json:"ovw,omitempty"` // overwrite destination
	DeleteSrc    bool `json:"dls,omitempty"` // remove source when (and after) successfully promoting
	// explicit request _not_ to treat the source as a potential file share
	// and _not_ to try to auto-detect if it is;
	// (auto-detection takes time, etc.)
	SrcIsNotFshare bool `json:"notshr,omitempty"` // the source is not a file share equally accessible by all targets
}

common part that's used in `api.PromoteArgs` and `PromoteParams`(server side), both

type PromoteParams

type PromoteParams struct {
	Bck         *Bck       // destination bucket
	Cksum       *cos.Cksum // checksum to validate
	Xact        Xact       // responsible xaction
	PromoteArgs            // all of the above
}

data path: control structures and types

type Providers

type Providers map[string]Namespaces

interface to Get current BMD instance (for implementation, see ais/bucketmeta.go)

type PutObjectParams

type PutObjectParams struct {
	Reader     io.ReadCloser
	Cksum      *cos.Cksum // checksum to check
	Atime      time.Time
	Xact       Xact
	WorkTag    string // (=> work fqn)
	OWT        cmn.OWT
	SkipEncode bool // don't run erasure-code when finalizing
}

data path: control structures and types

func AllocPutObjParams

func AllocPutObjParams() (a *PutObjectParams)

type QuiCB

type QuiCB func(elapsed time.Duration) QuiRes // see enum below

type QuiRes

type QuiRes int

type RMD

type RMD struct {
	Ext       any      `json:"ext,omitempty"` // within meta-version extensions
	Resilver  string   `json:"resilver,omitempty"`
	TargetIDs []string `json:"target_ids,omitempty"`
	Version   int64    `json:"version"`
}

RMD aka "rebalance metadata" is used to distribute information for the next rebalance.

func (*RMD) JspOpts

func (*RMD) JspOpts() jsp.Options

type RemAis

type RemAis struct {
	URL   string `json:"url"`
	Alias string `json:"alias"`
	UUID  string `json:"uuid"` // Smap.UUID
	Smap  *Smap  `json:"smap"`
}

type Remotes

type Remotes struct {
	A   []*RemAis `json:"a"`
	Ver int64     `json:"ver"`
}

type Slistener

type Slistener interface {
	String() string
	ListenSmapChanged()
}

Smap on-change listeners

type Smap

type Smap struct {
	Ext          any     `json:"ext,omitempty"`
	Pmap         NodeMap `json:"pmap"` // [pid => Snode]
	Primary      *Snode  `json:"proxy_si"`
	Tmap         NodeMap `json:"tmap"`          // [tid => Snode]
	UUID         string  `json:"uuid"`          // assigned once at creation time and never change
	CreationTime string  `json:"creation_time"` // creation timestamp
	Version      int64   `json:"version,string"`
}

func (*Smap) Compare

func (m *Smap) Compare(other *Smap) (uuid string, sameOrigin, sameVersion, eq bool)

func (*Smap) CompareTargets

func (m *Smap) CompareTargets(other *Smap) (equal bool)

func (*Smap) Count

func (m *Smap) Count() int

func (*Smap) CountActivePs added in v1.3.16

func (m *Smap) CountActivePs() (count int)

func (*Smap) CountActiveTs added in v1.3.16

func (m *Smap) CountActiveTs() (count int)

func (*Smap) CountNonElectable

func (m *Smap) CountNonElectable() (count int)

func (*Smap) CountProxies

func (m *Smap) CountProxies() int

func (*Smap) CountTargets

func (m *Smap) CountTargets() int

func (*Smap) DefaultICSize

func (*Smap) DefaultICSize() int

func (*Smap) GetActiveNode added in v1.3.16

func (m *Smap) GetActiveNode(sid string) (si *Snode)

(convenient, slightly redundant)

func (*Smap) GetNode

func (m *Smap) GetNode(id string) *Snode

func (*Smap) GetProxy

func (m *Smap) GetProxy(pid string) *Snode

func (*Smap) GetRandProxy

func (m *Smap) GetRandProxy(excludePrimary bool) (si *Snode, err error)

func (*Smap) GetRandTarget

func (m *Smap) GetRandTarget() (tsi *Snode, err error)

func (*Smap) GetTarget

func (m *Smap) GetTarget(sid string) *Snode

func (*Smap) HasActiveTargetPeers added in v1.3.16

func (m *Smap) HasActiveTargetPeers() bool

func (*Smap) ICCount

func (m *Smap) ICCount() int

func (*Smap) InMaintOrDecomm added in v1.3.16

func (m *Smap) InMaintOrDecomm(si *Snode) (ok bool)

given Snode, check (usually, the current) Smap that it is present _and_ InMaintOrDecomm (see also GetActiveNode)

func (*Smap) InitDigests

func (m *Smap) InitDigests()

func (*Smap) IsDuplicate

func (m *Smap) IsDuplicate(nsi *Snode) (osi *Snode, err error)

func (*Smap) IsIC

func (m *Smap) IsIC(psi *Snode) (ok bool)

func (*Smap) IsPrimary

func (m *Smap) IsPrimary(si *Snode) bool

func (*Smap) JspOpts

func (*Smap) JspOpts() jsp.Options

func (*Smap) NewTmap

func (m *Smap) NewTmap(tids []string) (tmap NodeMap, err error)

func (*Smap) NonElectable

func (m *Smap) NonElectable(psi *Snode) (ok bool)

func (*Smap) StrIC

func (m *Smap) StrIC(node *Snode) string

func (*Smap) String

func (m *Smap) String() string

func (*Smap) StringEx

func (m *Smap) StringEx() string

type SmapListeners

type SmapListeners interface {
	Reg(sl Slistener)
	Unreg(sl Slistener)
}

type Snap added in v1.3.16

type Snap struct {
	// xaction-specific stats counters
	Ext any `json:"ext"`

	// common static props
	StartTime time.Time `json:"start-time"`
	EndTime   time.Time `json:"end-time"`
	Bck       cmn.Bck   `json:"bck"`
	SrcBck    cmn.Bck   `json:"src-bck"`
	DstBck    cmn.Bck   `json:"dst-bck"`
	ID        string    `json:"id"`
	Kind      string    `json:"kind"`

	// rebalance-only
	RebID int64 `json:"glob.id,string"`

	// common runtime: stats counters (above) and state
	Stats    Stats `json:"stats"`
	AbortedX bool  `json:"aborted"`
	IdleX    bool  `json:"is_idle"`
}

func (*Snap) Finished added in v1.3.16

func (snp *Snap) Finished() bool

func (*Snap) IsAborted added in v1.3.16

func (snp *Snap) IsAborted() bool

func (*Snap) IsIdle added in v1.3.16

func (snp *Snap) IsIdle() bool

func (*Snap) Running added in v1.3.16

func (snp *Snap) Running() bool

func (*Snap) Started added in v1.3.16

func (snp *Snap) Started() bool

type Snode

type Snode struct {
	Ext        any        `json:"ext,omitempty"` // within meta-version extensions
	LocalNet   *net.IPNet `json:"-"`
	PubNet     NetInfo    `json:"public_net"`        // cmn.NetPublic
	DataNet    NetInfo    `json:"intra_data_net"`    // cmn.NetIntraData
	ControlNet NetInfo    `json:"intra_control_net"` // cmn.NetIntraControl
	DaeType    string     `json:"daemon_type"`       // "target" or "proxy"
	DaeID      string     `json:"daemon_id"`

	Flags cos.BitFlags `json:"flags"` // enum { SnodeNonElectable, SnodeIC, ... }
	// contains filtered or unexported fields
}

Snode - a node (gateway or target) in a cluster

func HrwIC

func HrwIC(smap *Smap, uuid string) (pi *Snode, err error)

func HrwProxy

func HrwProxy(smap *Smap, idToSkip string) (pi *Snode, err error)

func HrwTarget

func HrwTarget(uname string, smap *Smap) (si *Snode, err error)

func HrwTargetAll

func HrwTargetAll(uname string, smap *Smap) (si *Snode, err error)

Include targets in maintenance

func HrwTargetTask

func HrwTargetTask(uuid string, smap *Smap) (si *Snode, err error)

Returns a target for a given task. E.g. usage: list objects in a cloud bucket (we want only one target to do it).

func NewSnode

func NewSnode(id, daeType string, publicNet, intraControlNet, intraDataNet NetInfo) (snode *Snode)

func (*Snode) Clone

func (d *Snode) Clone() *Snode

func (*Snode) Digest

func (d *Snode) Digest() uint64

func (*Snode) Equals

func (d *Snode) Equals(o *Snode) (eq bool)

func (*Snode) ID

func (d *Snode) ID() string

func (*Snode) InMaintOrDecomm added in v1.3.16

func (d *Snode) InMaintOrDecomm() bool

node flags

func (*Snode) Init

func (d *Snode) Init(id, daeType string)

func (*Snode) IsProxy

func (d *Snode) IsProxy() bool

func (*Snode) IsTarget

func (d *Snode) IsTarget() bool

func (*Snode) Name

func (d *Snode) Name() string

func (*Snode) SetName

func (d *Snode) SetName()

func (*Snode) String

func (d *Snode) String() string

func (*Snode) StringEx

func (d *Snode) StringEx() string

func (*Snode) Type

func (d *Snode) Type() string

func (*Snode) URL

func (d *Snode) URL(network string) string

func (*Snode) Validate

func (d *Snode) Validate() error

type Sowner

type Sowner interface {
	Get() (smap *Smap)
	Listeners() SmapListeners
}

interface to Get current cluster-map instance

type Stats added in v1.3.16

type Stats struct {
	Objs     int64 `json:"loc-objs,string"`  // locally processed
	Bytes    int64 `json:"loc-bytes,string"` //
	OutObjs  int64 `json:"out-objs,string"`  // transmit
	OutBytes int64 `json:"out-bytes,string"` //
	InObjs   int64 `json:"in-objs,string"`   // receive
	InBytes  int64 `json:"in-bytes,string"`
}

type Target

type Target interface {
	TargetLoc

	// (for intra-cluster data-net comm - no streams)
	DataClient() *http.Client

	CompareObjects(ctx context.Context, lom *LOM) (equal bool, errCode int, err error)

	// core object (+ PutObject above)
	FinalizeObj(lom *LOM, workFQN string, xctn Xact) (errCode int, err error)
	EvictObject(lom *LOM) (errCode int, err error)
	DeleteObject(lom *LOM, evict bool) (errCode int, err error)
	CopyObject(lom *LOM, params *CopyObjectParams, dryRun bool) (int64, error)
	GetCold(ctx context.Context, lom *LOM, owt cmn.OWT) (errCode int, err error)
	Promote(params PromoteParams) (errCode int, err error)
	HeadObjT2T(lom *LOM, si *Snode) bool
}

all of the above; for implementations, see `ais/tgtimpl.go` and `ais/htrun.go`

type TargetExt added in v1.3.16

type TargetExt interface {
	Target

	// misc
	BMDVersionFixup(r *http.Request, bck ...cmn.Bck)
}

type TargetLoc added in v1.3.16

type TargetLoc interface {
	TargetPut

	// backend
	Backend(*Bck) BackendProvider

	// FS health and Health
	FSHC(err error, path string)
	Health(si *Snode, timeout time.Duration, query url.Values) (body []byte, errCode int, err error)
}

local target node

var (
	T TargetLoc
)

type TargetPut added in v1.3.16

type TargetPut interface {
	NodeMemCap

	// local PUT
	PutObject(lom *LOM, params *PutObjectParams) (err error)
}

a node that can also write objects

type Upon

type Upon int

type Xact

type Xact interface {
	Run(*sync.WaitGroup)
	ID() string
	Kind() string
	Bck() *Bck
	FromTo() (*Bck, *Bck)
	StartTime() time.Time
	EndTime() time.Time
	Finished() bool
	Running() bool
	IsAborted() bool
	AbortErr() error
	AbortedAfter(time.Duration) error
	ChanAbort() <-chan error
	Quiesce(time.Duration, QuiCB) QuiRes
	Result() (any, error)

	Snap() *Snap // (struct below)

	// reporting: log, err
	String() string
	Name() string

	// modifiers
	Finish(error)
	Abort(error) bool
	AddNotif(n Notif)

	// common stats
	Objs() int64
	ObjsAdd(int, int64)    // locally processed
	OutObjsAdd(int, int64) // transmit
	InObjsAdd(int, int64)  // receive
	InBytes() int64
	OutBytes() int64
}

Directories

Path Synopsis
Package mock provides a variety of mock implementations used for testing.
Package mock provides a variety of mock implementations used for testing.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL