apc

package
v1.3.19 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 29, 2023 License: MIT Imports: 10 Imported by: 7

Documentation

Overview

Package apc: API messages and constants

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package apc: API constant and control messages

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2021-2023, NVIDIA CORPORATION. All rights reserved.

Package apc: API messages and constants

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source 
const (
	// object level
	AceGET     = AccessAttrs(1) << iota
	AceObjHEAD // permission to get object props
	AcePUT
	AceAPPEND
	AceObjDELETE
	AceObjMOVE
	AcePromote
	// permission to overwrite objects that were previously read from:
	// a) any remote backend that is currently not configured as the bucket's backend
	// b) HTPP ("ht://") since it's not writable
	AceDisconnectedBackend
	// bucket metadata
	AceBckHEAD   // get bucket props and ACL
	AceObjLIST   // list objects in a bucket
	AcePATCH     // set bucket props
	AceBckSetACL // set bucket permissions
	// cluster level
	AceListBuckets
	AceShowCluster
	AceCreateBucket
	AceDestroyBucket
	AceMoveBucket
	AceAdmin
	// note: must be the last one
	AceMax
)

ACL aka access permissions

View Source 
const (
	// encompasses all ACEs, current and future
	AccessAll      = AccessAttrs(^uint64(0))
	AllowAllAccess = "su"

	// read-only and read-write access to bucket
	AccessRO             = AceGET | AceObjHEAD | AceListBuckets | AceBckHEAD | AceObjLIST
	AllowReadOnlyAccess  = "ro"
	AccessRW             = AccessRO | AcePUT | AceAPPEND | AceObjDELETE | AceObjMOVE
	AllowReadWriteAccess = "rw"

	AccessNone = AccessAttrs(0)

	// permission to perform cluster-level ops
	AccessCluster = AceListBuckets | AceCreateBucket | AceDestroyBucket | AceMoveBucket | AceAdmin
)

derived (convenience) constants

View Source 
const (
	ActCreateBck   = "create-bck"  // NOTE: compare w/ ActAddRemoteBck below
	ActDestroyBck  = "destroy-bck" // destroy bucket data and metadata
	ActSetBprops   = "set-bprops"
	ActResetBprops = "reset-bprops"

	ActSummaryBck = "summary-bck"

	ActECEncode  = "ec-encode" // erasure code a bucket
	ActECGet     = "ec-get"    // erasure decode objects
	ActECPut     = "ec-put"    // erasure encode objects
	ActECRespond = "ec-resp"   // respond to other targets' EC requests

	ActCopyBck = "copy-bck"
	ActETLBck  = "etl-bck"

	ActETLInline = "etl-inline"

	ActDsort    = "dsort"
	ActDownload = "download"

	ActMakeNCopies = "make-n-copies"
	ActPutCopies   = "put-copies"

	ActRebalance = "rebalance"
	ActMoveBck   = "move-bck"

	ActResilver = "resilver"

	ActElection = "election"

	ActLRU          = "lru"
	ActStoreCleanup = "cleanup-store"

	ActEvictRemoteBck = "evict-remote-bck" // evict remote bucket's data
	ActInvalListCache = "inval-listobj-cache"
	ActList           = "list"
	ActLoadLomCache   = "load-lom-cache"
	ActNewPrimary     = "new-primary"
	ActPromote        = "promote"
	ActRenameObject   = "rename-obj"

	ActResetStats  = "reset-stats"
	ActResetConfig = "reset-config"
	ActSetConfig   = "set-config"

	ActShutdownCluster = "shutdown" // see also: ActShutdownNode

	// multi-object (via `ListRange`)
	ActCopyObjects     = "copy-listrange"
	ActDeleteObjects   = "delete-listrange"
	ActETLObjects      = "etl-listrange"
	ActEvictObjects    = "evict-listrange"
	ActPrefetchObjects = "prefetch-listrange"
	ActArchive         = "archive" // see ArchiveMsg

	ActAttachRemAis = "attach"
	ActDetachRemAis = "detach"

	// Node maintenance & cluster membership (see also ActRmNodeUnsafe below)
	ActStartMaintenance = "start-maintenance" // put into maintenance state
	ActStopMaintenance  = "stop-maintenance"  // cancel maintenance state
	ActShutdownNode     = "shutdown-node"     // shutdown node
	ActDecommissionNode = "decommission-node" // start rebalance and, when done, remove node from Smap

	ActDecommissionCluster = "decommission" // decommission all nodes in the cluster (cleanup system data)

	ActAdminJoinTarget = "admin-join-target"
	ActSelfJoinTarget  = "self-join-target"
	ActAdminJoinProxy  = "admin-join-proxy"
	ActSelfJoinProxy   = "self-join-proxy"
	ActKeepaliveUpdate = "keepalive-update"

	// IC
	ActSendOwnershipTbl  = "ic-send-own-tbl"
	ActListenToNotif     = "watch-xaction"
	ActMergeOwnershipTbl = "ic-merge-own-tbl"
	ActRegGlobalXaction  = "reg-global-xaction"
)

ActMsg.Action includes Xaction.Kind == ActMsg.Action (when the action is asynchronous)

View Source 
const (
	ActAddRemoteBck   = "add-remote-bck" // add to BMD existing remote bucket, usually on the fly
	ActRmNodeUnsafe   = "rm-unsafe"      // primary => the node to be removed
	ActStartGFN       = "start-gfn"      // get-from-neighbor
	ActStopGFN        = "stop-gfn"       // off
	ActCleanupMarkers = "cleanup-markers"
)

internal use

View Source 
const (
	// Actions on mountpaths (/v1/daemon/mountpaths)
	ActMountpathAttach  = "attach-mp"
	ActMountpathEnable  = "enable-mp"
	ActMountpathDetach  = "detach-mp"
	ActMountpathDisable = "disable-mp"

	// Actions on xactions
	ActXactStop  = Stop
	ActXactStart = Start

	// auxiliary
	ActTransient = "transient" // transient - in-memory only
)
View Source 
const (
	ActBegin  = "begin"
	ActCommit = "commit"
	ActAbort  = "abort"
)

xaction begin-commit phases

View Source 
const (
	NodeMaintenance  = "maintenance"
	NodeDecommission = "decommission"
)
View Source 
const (
	CompressAlways = "always"
	CompressNever  = "never"
)

Compression enum

View Source 
const (
	Proxy  = "proxy"
	Target = "target"
)
View Source 
const (
	DeploymentK8s = "K8s"
	DeploymentDev = "dev"
)

deployment types

View Source 
const (
	DefaultTimeout = time.Duration(-1)
	LongTimeout    = time.Duration(-2)
)

timeouts for intra-cluster requests

View Source 
const (
	HeaderPrefix = "ais-"

	// Bucket props headers.
	HdrBucketProps      = HeaderPrefix + "bucket-props"       // => cmn.BucketProps
	HdrBucketSumm       = HeaderPrefix + "bucket-summ"        // => cmn.BsummResult (see also: QparamFltPresence)
	HdrBucketVerEnabled = HeaderPrefix + "versioning-enabled" // Enable/disable object versioning in a bucket.
	HdrBucketCreated    = HeaderPrefix + "created"            // Bucket creation time.
	HdrBackendProvider  = HeaderPrefix + "provider"           // ProviderAmazon et al. - see cmn/bck.go.

	// including BucketProps.Extra.AWS
	HdrS3Region   = HeaderPrefix + "cloud_region"
	HdrS3Endpoint = HeaderPrefix + "endpoint"
	HdrS3Profile  = HeaderPrefix + "profile"

	// including BucketProps.Extra.HTTP
	HdrOrigURLBck = HeaderPrefix + "original-url"

	// remote AIS
	HdrRemAisUUID  = HeaderPrefix + "remote-ais-uuid"
	HdrRemAisAlias = HeaderPrefix + "remote-ais-alias"
	HdrRemAisURL   = HeaderPrefix + "remote-ais-url"

	HdrRemoteOffline = HeaderPrefix + "remote-offline" // When accessing cached remote bucket with no backend connectivity.

	// Object props headers
	HdrObjCksumType = HeaderPrefix + "checksum-type"  // Checksum type, one of SupportedChecksums().
	HdrObjCksumVal  = HeaderPrefix + "checksum-value" // Checksum value.
	HdrObjAtime     = HeaderPrefix + "atime"          // Object access time.
	HdrObjCustomMD  = HeaderPrefix + "custom-md"      // Object custom metadata.
	HdrObjVersion   = HeaderPrefix + "version"        // Object version/generation - ais or cloud.

	// Archive filename and format (mime type)
	HdrArchpath = HeaderPrefix + "archpath"
	HdrArchmime = HeaderPrefix + "archmime"

	// Append object header.
	HdrAppendHandle = HeaderPrefix + "append-handle"

	// api.PutApndArchArgs message flags
	HdrPutApndArchFlags = HeaderPrefix + "pine"

	// Query objects handle header.
	HdrHandle = HeaderPrefix + "query-handle"

	// Reverse proxy headers.
	HdrNodeID  = HeaderPrefix + "node-id"
	HdrNodeURL = HeaderPrefix + "node-url"

	// uptimes, respectively
	HdrNodeUptime    = HeaderPrefix + "node-uptime"
	HdrClusterUptime = HeaderPrefix + "cluster-uptime"
)

Header Key conventions:

  • starts with a prefix "ais-",
  • all words separated with "-": no dots and underscores.
View Source 
const (
	HdrAuthorization         = "Authorization" // https://developer.mozilla.org/en-US/docs/Web/HTTP/Hdrs/Authorization
	AuthenticationTypeBearer = "Bearer"
)

AuthN consts

View Source 
const (
	HdrCallerID          = HeaderPrefix + "caller-id" // Marker of intra-cluster request.
	HdrT2TPutterID       = HeaderPrefix + "putter-id" // DaemonID of the target that performs intra-cluster PUT
	HdrCallerName        = HeaderPrefix + "caller-name"
	HdrCallerSmapVersion = HeaderPrefix + "caller-smap-ver"

	HdrXactionID = HeaderPrefix + "xaction-id"

	// Stream related headers.
	HdrSessID   = HeaderPrefix + "session-id"
	HdrCompress = HeaderPrefix + "compress" // LZ4Compression, etc.

	// Promote(dir)
	HdrPromoteNamesHash = HeaderPrefix + "promote-names-hash"
	HdrPromoteNamesNum  = HeaderPrefix + "promote-names-num"
)

Internally used headers

View Source 
const (
	LocationPropSepa = ":"
	LsPropsSepa      = ","
)
View Source 
const (
	// Applies to objects from the buckets with remote backends (e.g., to optimize-out listing remotes)
	// See related Flt* enum
	LsObjCached = 1 << iota

	LsAll      // include misplaced objects and replicas
	LsDeleted  // include obj-s marked for deletion (TODO)
	LsArchDir  // expand archives as directories
	LsNameOnly // return only object names and statuses (for faster listing)
	LsNameSize // same as above plus size

	// LsDontHeadRemote tells AIS _not_ to execute HEAD request on the remote bucket.
	// The reasons may include cleanup/eviction of any kind, prior knowledge that the bucket
	// must simply exist in AIS, and more.
	// See also:
	// * `cmn/feat/feat.go` source, and the (configurable) capability
	//    to disable on-the-fly creation of remote buckets altogether.
	LsDontHeadRemote

	// LsTryHeadRemote is introduced primarily to support GCP buckets with
	// ACL policies that allow public _anonymous_ access.
	//
	// It appears that sometimes those policies do honor HEAD(bucket),
	// while other times they don't, failing the request with 401 or 403 status.
	// See also:
	// * at https://cloud.google.com/storage/docs/access-control/making-data-public
	LsTryHeadRemote

	// To list remote buckets that, if not be present in AIS, shall not be added to AIS
	// (TODO: reserved for future use)
	LsDontAddRemote

	// cache list-objects results and use this cache to speed-up
	UseListObjsCache

	// For remote buckets - list only remote props (aka `wantOnlyRemote`). When false,
	// the default that's being used is: `WantOnlyRemoteProps` - see below.
	// When true, the request gets executed in a pass-through fashion whereby a single ais target
	// simply forwards it to the associated remote backend and delivers the results as is to the
	// requesting proxy and, subsequently, to client.
	LsWantOnlyRemoteProps

	// List bucket entries without recursion (POSIX-wise). Note that the result in this case
	// will include matching directories.
	// TODO: works only with AIS bucket and only via S3 (no CLI yet). More exactly:
	// - update AIS CLI to support non-recursive list-objects operation
	// - when listing remote bucket, call backend (`Backend()`) to list non-recursively
	LsNoRecursion
)

LsoMsg flags

View Source 
const (
	DefaultPageSizeAIS   = 10000
	DefaultPageSizeCloud = 1000
)

List objects default page size

View Source 
const (
	// Status
	LocOK = iota
	LocMisplacedNode
	LocMisplacedMountpath
	LocIsCopy
	LocIsCopyMissingObj

	// LsoEntry Flags
	EntryIsCached  = 1 << (EntryStatusBits + 1)
	EntryInArch    = 1 << (EntryStatusBits + 2)
	EntryIsDir     = 1 << (EntryStatusBits + 3)
	EntryIsArchive = 1 << (EntryStatusBits + 4)
)
View Source 
const (
	EntryStatusBits = 5                          // N bits
	EntryStatusMask = (1 << EntryStatusBits) - 1 // mask for N low bits
)

ObjEntry.Flags field

View Source 
const (
	GetPropsName     = "name"
	GetPropsSize     = "size"
	GetPropsVersion  = "version"
	GetPropsChecksum = "checksum"
	GetPropsAtime    = "atime"
	GetPropsCached   = "cached"
	GetPropsStatus   = "status"
	GetPropsCopies   = "copies"
	GetPropsEC       = "ec"
	GetPropsCustom   = "custom"
	GetPropsLocation = "location" // advanced usage
)

LsoMsg and HEAD(object) enum (NOTE: compare with `cmn.ObjectProps`)

View Source 
const (
	AIS   = "ais"
	AWS   = "aws"
	Azure = "azure"
	GCP   = "gcp"
	HDFS  = "hdfs"
	HTTP  = "ht"

	AllProviders = "ais, aws (s3://), gcp (gs://), azure (az://), hdfs://, ht://" // NOTE: must include all

	NsUUIDPrefix = '@' // BEWARE: used by on-disk layout
	NsNamePrefix = '#' // BEWARE: used by on-disk layout

	BckProviderSeparator = "://"
	BckObjnameSeparator  = "/"

	// scheme://
	DefaultScheme = "https"
	GSScheme      = "gs"
	S3Scheme      = "s3"
	AZScheme      = "az"
	AISScheme     = "ais"
)

Backend Provider enum

View Source 
const (
	ArchAppend = 1 << iota
	ArchAppendIfExist
)

api.PutApndArchArgs flags

View Source 
const (
	QparamWhat = "what" // "smap" | "bmd" | "config" | "stats" | "xaction" ... (enum below)

	QparamProps = "props" // e.g. "checksum, size"|"atime, size"|"cached"|"bucket, size"| ...

	QparamUUID    = "uuid"     // xaction
	QparamJobID   = "jobid"    // job
	QparamETLName = "etl_name" // etl

	QparamRegex      = "regex"       // dsort: list regex
	QparamOnlyActive = "only_active" // dsort: list only active

	// remove existing custom keys and store new custom metadata
	// NOTE: making an s/_/-/ naming exception because of the namesake CLI usage
	QparamNewCustom = "set-new-custom"

	// Main bucket query params.
	QparamProvider  = "provider" // aka backend provider or, simply, backend
	QparamNamespace = "namespace"

	// e.g., usage: copy bucket
	QparamBckTo = "bck_to"

	// Do not add remote bucket to cluster's BMD e.g. when checking existence
	// via api.HeadBucket
	// By default, when existence of a remote buckets is confirmed the bucket's
	// metadata gets automatically (and transactionally) added to the cluster's BMD.
	// This query parameter can be used to override the default behavior.
	QparamDontAddRemote = "dont_add_remote_bck_md"

	// Add remote bucket to BMD _unconditionally_ and without executing HEAD request
	// (to check access and load the bucket's properties)
	// NOTE: usage is limited to setting up bucket properties with alternative
	// profile and/or endpoint
	// See also:
	// - `LsDontHeadRemote`
	// - docs/bucket.md
	// - docs/cli/aws_profile_endpoint.md
	QparamDontHeadRemote = "dont_head_remote_bck"

	// When evicting, keep remote bucket in BMD (i.e., evict data only)
	QparamKeepRemote = "keep_bck_md"

	// When summarizing via a (blocking) api.GetBucketInfo call, provide remote stats as well
	// NOTE: to be used with caution! depending on remote bucket size and network speed
	//       the waiting time may be significant
	QparamCountRemoteObjs = "count_remote_objs"

	// NOTE: "presence" in a given cluster shall not be be confused with "existence" (possibly, remote).
	// See also:
	// - Flt* enum below
	// - ListObjsMsg flags, docs/providers.md (for terminology)
	QparamFltPresence = "presence"

	// Object related query params.
	QparamAppendType   = "append_type"
	QparamAppendHandle = "append_handle"

	// HTTP bucket support.
	QparamOrigURL = "original_url"

	// Get logs
	QparamLogSev  = "severity" // see { LogInfo, ...} enum
	QparamLogOff  = "offset"
	QparamAllLogs = "all"

	// Archive filename and format (mime type)
	QparamArchpath = "archpath"
	QparamArchmime = "archmime"

	// Skip loading existing object's metadata, in part to
	// compare its Checksum and update its existing Version (if exists).
	// Can be used to reduce PUT latency when:
	// - we massively write new content into a bucket, and/or
	// - we simply don't care.
	QparamSkipVC = "skip_vc"

	// force operation
	// used to overcome certain restrictions, e.g.:
	// - shutdown the primary and the entire cluster
	// - attach invalid mountpath
	QparamForce = "frc"
)

URL Query "?name1=val1&name2=..." User query params.

View Source 
const (
	FltExists         = iota // (object | bucket) exists inside and/or outside cluster
	FltExistsNoProps         // same as above but no need to return props/info
	FltPresent               // bucket: is present | object: present and properly located
	FltPresentNoProps        // same as above but no need to return props/info
	FltPresentCluster        // objects: present anywhere/anyhow _in_ the cluster as: replica, ec-slices, misplaced
	FltExistsOutside         // not present - exists _outside_ cluster
)

QparamFltPresence enum.

Descibes both buckets and objects with respect to their existence/presence (or non-existence/non-presence) in AIS cluster.

"FltPresent*" refers to availability ("presence") in the cluster. For details, see the values and comments below.

Remote object or bucket that is currently not present can still be accessed with the very first access making it "present", etc.

View Source 
const (
	AppendOp = "append"
	FlushOp  = "flush"
)

QparamAppendType enum.

View Source 
const (
	TaskStart  = Start
	TaskStatus = "status"
	TaskResult = "result"
)

QparamTaskAction enum.

View Source 
const (
	QparamHealthReadiness = "readiness" // to be used by external watchdogs (e.g. K8s)
	QparamAskPrimary      = "apr"       // true: the caller is directing health request to primary
	QparamPrimaryReadyReb = "prr"       // true: check whether primary is ready to start rebalancing cluster
)

health

View Source 
const (
	QparamProxyID          = "pid" // ID of the redirecting proxy.
	QparamPrimaryCandidate = "can" // ID of the candidate for the primary proxy.
	QparamPrepare          = "prp" // true: request belongs to the "prepare" phase of the primary proxy election
	QparamNonElectable     = "nel" // true: proxy is non-electable for the primary role
	QparamUnixTime         = "utm" // Unix time since 01/01/70 UTC (nanoseconds)
	QparamIsGFNRequest     = "gfn" // true if the request is a Get-From-Neighbor
	QparamSilent           = "sln" // true: destination should not log errors (HEAD request)
	QparamRebStatus        = "rbs" // true: get detailed rebalancing status
	QparamRebData          = "rbd" // true: get EC rebalance data (pulling data if push way fails)
	QparamTaskAction       = "tac" // "start", "status", "result"
	QparamClusterInfo      = "cii" // true: /Health to return cluster info and status
	QparamOWT              = "owt" // object write transaction enum { OwtPut, ..., OwtGet* }

	QparamDontResilver = "dntres" // true: do not resilver data off of mountpaths that are being disabled/detached

	// dsort
	QparamTotalCompressedSize       = "tcs"
	QparamTotalInputShardsExtracted = "tise"
	QparamTotalUncompressedSize     = "tunc"

	// 2PC transactions - control plane
	QparamNetwTimeout  = "xnt" // [begin, start-commit] timeout
	QparamHostTimeout  = "xht" // [begin, txn-done] timeout
	QparamWaitMetasync = "xwm" // true: wait for metasync (used only when there's an alternative)

	// promote
	QparamConfirmFshare = "confirm-fshr" // confirm file share
	QparamActNoXact     = "act-no-xact"  // execute synchronously, i.e. without xaction

	// Notification target's node ID (usually, the node that initiates the operation).
	QparamNotifyMe = "nft"
)

Internal query params.

View Source 
const (
	// cluster meta
	WhatSmap = "smap"
	WhatBMD  = "bmd"
	// config
	WhatNodeConfig    = "config" // query specific node for (cluster config + overrides, local config)
	WhatClusterConfig = "cluster_config"
	// stats
	WhatNodeStats          = "stats"
	WhatNodeStatsAndStatus = "status"
	WhatMetricNames        = "metrics"
	WhatDiskStats          = "disk"
	// assorted
	WhatMountpaths = "mountpaths"
	WhatRemoteAIS  = "remote"
	WhatSmapVote   = "smapvote"
	WhatSysInfo    = "sysinfo"
	WhatTargetIPs  = "target_ips" // comma-separated list of all target IPs (compare w/ GetWhatSnode)
	// log
	WhatLog = "log"
	// xactions
	WhatOneXactStatus   = "status"      // IC status by uuid (returns a single matching xaction or none)
	WhatAllXactStatus   = "status_all"  // ditto - all matching xactions
	WhatXactStats       = "getxstats"   // stats: xaction by uuid
	WhatQueryXactStats  = "qryxstats"   // stats: all matching xactions
	WhatAllRunningXacts = "running_all" // e.g. e.g.: put-copies[D-ViE6HEL_j] list[H96Y7bhR2s] ...
	// internal
	WhatSnode    = "snode"
	WhatICBundle = "ic_bundle"
)

QparamWhat enum.

View Source 
const (
	LogInfo = "info"
	LogWarn = "warning"
	LogErr  = "error"
)

QparamLogSev enum.

View Source 
const (
	// l1
	Version = "v1"
	// l2
	Buckets   = "buckets"
	Objects   = "objects"
	EC        = "ec"
	Download  = "download"
	Daemon    = "daemon"
	Cluster   = "cluster"
	Tokens    = "tokens"
	Metasync  = "metasync"
	Health    = "health"
	Vote      = "vote"
	ObjStream = "objstream"
	MsgStream = "msgstream"
	Reverse   = "reverse"
	Rebalance = "rebalance"
	Xactions  = "xactions"
	S3        = "s3"
	Txn       = "txn"      // 2PC
	Notifs    = "notifs"   // intra-cluster notifications
	Users     = "users"    // AuthN
	Clusters  = "clusters" // AuthN
	Roles     = "roles"    // AuthN
	IC        = "ic"       // information center

	Voteres  = "result"
	VoteInit = "init"
	PriStop  = "primary-stopping"

	// (see the corresponding action messages above)
	Keepalive = "keepalive"
	AdminJoin = "join-by-admin" // when node is joined by admin ("manual join")
	SelfJoin  = "autoreg"       // auto-join cluster at startup

	// target
	Mountpaths = "mountpaths"

	// common
	Init     = "init"
	Start    = "start"
	Stop     = "stop"
	Abort    = "abort"
	Sort     = "sort"
	Finished = "finished"
	Progress = "progress"

	// dsort, dloader, query
	Metrics     = "metrics"
	Records     = "records"
	Shards      = "shards"
	FinishedAck = "finished_ack"
	List        = "list"
	Remove      = "remove"
	Next        = "next"
	Peek        = "peek"
	Discard     = "discard"
	WorkerOwner = "worker" // TODO: it should be removed once get-next-bytes endpoint is ready

	// ETL
	ETL        = "etl"
	ETLInfo    = "info"
	ETLList    = List
	ETLLogs    = "logs"
	ETLObject  = "_object"
	ETLStop    = Stop
	ETLStart   = Start
	ETLHealth  = "health"
	ETLMetrics = "metrics"
)

RESTful URL path: l1/l2/l3

View Source 
const (
	WriteImmediate = WritePolicy("immediate") // immediate write (default)
	WriteDelayed   = WritePolicy("delayed")   // cache and flush when not accessed for a while (lom_cache_hk.go)
	WriteNever     = WritePolicy("never")     // transient - in-memory only

	WriteDefault = WritePolicy("") // same as `WriteImmediate` - see IsImmediate() below
)
View Source 
const HdrError = "Hdr-Error"
View Source 
const LZ4Compression = "lz4"

sent via req.Header.Set(apc.HdrCompress, LZ4Compression) (alternative to lz4 compressions upon popular request)

View Source 
const (
	RemAisDefunct = "defunct" // uuid configured offline
)
View Source 
const (
	SyncSmap = "syncsmap"
)

RESTful l3, internal use

Variables

View Source 
var (
	GetPropsMinimal      = []string{GetPropsName, GetPropsSize, GetPropsCached}
	GetPropsDefaultCloud = []string{GetPropsName, GetPropsSize, GetPropsCached,
		GetPropsChecksum, GetPropsVersion, GetPropsCustom}

	GetPropsDefaultAIS = []string{GetPropsName, GetPropsSize, GetPropsChecksum, GetPropsAtime}
	GetPropsAll        = []string{GetPropsName, GetPropsSize, GetPropsChecksum, GetPropsAtime,
		GetPropsVersion, GetPropsCached, GetPropsStatus, GetPropsCopies, GetPropsEC, GetPropsCustom, GetPropsLocation}
)

NOTE: update when changing any of the above :NOTE

View Source 
var (
	URLPathS3 = urlpath(S3) // URLPath{[]string{S3}, S3}

	URLPathBuckets   = urlpath(Version, Buckets)
	URLPathObjects   = urlpath(Version, Objects)
	URLPathEC        = urlpath(Version, EC)
	URLPathNotifs    = urlpath(Version, Notifs)
	URLPathTxn       = urlpath(Version, Txn)
	URLPathXactions  = urlpath(Version, Xactions)
	URLPathIC        = urlpath(Version, IC)
	URLPathHealth    = urlpath(Version, Health)
	URLPathMetasync  = urlpath(Version, Metasync)
	URLPathRebalance = urlpath(Version, Rebalance)

	URLPathClu        = urlpath(Version, Cluster)
	URLPathCluProxy   = urlpath(Version, Cluster, Proxy)
	URLPathCluUserReg = urlpath(Version, Cluster, AdminJoin)
	URLPathCluAutoReg = urlpath(Version, Cluster, SelfJoin)
	URLPathCluKalive  = urlpath(Version, Cluster, Keepalive)
	URLPathCluDaemon  = urlpath(Version, Cluster, Daemon)
	URLPathCluSetConf = urlpath(Version, Cluster, ActSetConfig)
	URLPathCluAttach  = urlpath(Version, Cluster, ActAttachRemAis)
	URLPathCluDetach  = urlpath(Version, Cluster, ActDetachRemAis)

	URLPathDae          = urlpath(Version, Daemon)
	URLPathDaeProxy     = urlpath(Version, Daemon, Proxy)
	URLPathDaeSetConf   = urlpath(Version, Daemon, ActSetConfig)
	URLPathDaeAdminJoin = urlpath(Version, Daemon, AdminJoin)

	URLPathReverse    = urlpath(Version, Reverse)
	URLPathReverseDae = urlpath(Version, Reverse, Daemon)

	URLPathVote        = urlpath(Version, Vote)
	URLPathVoteInit    = urlpath(Version, Vote, Init)
	URLPathVoteProxy   = urlpath(Version, Vote, Proxy)
	URLPathVoteVoteres = urlpath(Version, Vote, Voteres)
	URLPathVotePriStop = urlpath(Version, Vote, PriStop)

	URLPathdSort        = urlpath(Version, Sort)
	URLPathdSortInit    = urlpath(Version, Sort, Init)
	URLPathdSortStart   = urlpath(Version, Sort, Start)
	URLPathdSortList    = urlpath(Version, Sort, List)
	URLPathdSortAbort   = urlpath(Version, Sort, Abort)
	URLPathdSortShards  = urlpath(Version, Sort, Shards)
	URLPathdSortRecords = urlpath(Version, Sort, Records)
	URLPathdSortMetrics = urlpath(Version, Sort, Metrics)
	URLPathdSortAck     = urlpath(Version, Sort, FinishedAck)
	URLPathdSortRemove  = urlpath(Version, Sort, Remove)

	URLPathDownload       = urlpath(Version, Download)
	URLPathDownloadAbort  = urlpath(Version, Download, Abort)
	URLPathDownloadRemove = urlpath(Version, Download, Remove)

	URLPathETL       = urlpath(Version, ETL)
	URLPathETLObject = urlpath(Version, ETL, ETLObject)

	URLPathTokens   = urlpath(Version, Tokens) // authn
	URLPathUsers    = urlpath(Version, Users)
	URLPathClusters = urlpath(Version, Clusters)
	URLPathRoles    = urlpath(Version, Roles)
)
View Source 
var Providers = cos.NewStrSet(AIS, GCP, AWS, Azure, HDFS, HTTP)
View Source 
var SupportedCompression = []string{CompressNever, CompressAlways}
View Source 
var SupportedWritePolicy = []string{string(WriteImmediate), string(WriteDelayed), string(WriteNever)}

Functions

func AccessOp 

func AccessOp(access AccessAttrs) string

func DisplayProvider 

func DisplayProvider(p string) string

func IsCloudProvider added in v1.3.16 

func IsCloudProvider(p string) bool

func IsFltNoProps 

func IsFltNoProps(v int) bool

func IsFltPresent 

func IsFltPresent(v int) bool

func IsProvider 

func IsProvider(p string) bool

func IsRemoteProvider added in v1.3.16 

func IsRemoteProvider(p string) bool

func IsValidCompression 

func IsValidCompression(c string) bool

func NormalizeProvider 

func NormalizeProvider(p string) string

func SupportedPermissions 

func SupportedPermissions() []string

verbs

func ToScheme 

func ToScheme(p string) string

Types

type AccessAttrs 

type AccessAttrs uint64

func StrToAccess 

func StrToAccess(accessStr string) (access AccessAttrs, err error)

func (AccessAttrs) Describe 

func (a AccessAttrs) Describe() string

func (AccessAttrs) Has 

func (a AccessAttrs) Has(perms AccessAttrs) bool

func (AccessAttrs) String 

func (a AccessAttrs) String() string

type ActMsg added in v1.3.16 

type ActMsg struct {
	Value  any    `json:"value"`  // action-specific and optional
	Action string `json:"action"` // ActShutdown, ActRebalance, and many more (see apc/const.go)
	Name   string `json:"name"`   // action-specific name (e.g., bucket name)
}

ActMsg is a JSON-formatted control structures used in a majority of API calls

func (*ActMsg) String added in v1.3.16 

func (msg *ActMsg) String() string

func (*ActMsg) StringEx added in v1.3.18 

func (msg *ActMsg) StringEx() string

type ActValRmNode 

type ActValRmNode struct {
	DaemonID          string `json:"sid"`
	SkipRebalance     bool   `json:"skip_rebalance"`
	RmUserData        bool   `json:"rm_user_data"`        // decommission-only
	KeepInitialConfig bool   `json:"keep_initial_config"` // ditto (to be able to restart a node from scratch)
	NoShutdown        bool   `json:"no_shutdown"`
}

ActMsg is a JSON-formatted control structures used in a majority of API calls

type ArchiveMsg added in v1.3.18 

type ArchiveMsg struct {
	TxnUUID     string `json:"-"`        // internal use
	FromBckName string `json:"-"`        // ditto
	ArchName    string `json:"archname"` // one of the archive.FileExtensions
	Mime        string `json:"mime"`     // user-specified mime type (NOTE: takes precedence if defined)
	ListRange
	InclSrcBname    bool `json:"isbn"` // include source bucket name into the names of archived objects
	AppendIfExists  bool `json:"aate"` // adding a list or a range of objects to an existing archive
	ContinueOnError bool `json:"coer"` // on err, keep running arc xaction in a any given multi-object transaction
}

ArchiveMsg contains the parameters (all except the destination bucket) for archiving mutiple objects as one of the supported archive.FileExtensions types at the specified (bucket) destination. See also: api.PutApndArchArgs -------------------- terminology --------------------- here and elsewhere "archive" is any (.tar, .tgz/.tar.gz, .zip, .tar.lz4) formatted object.

type BsummCtrlMsg added in v1.3.18 

type BsummCtrlMsg struct {
	UUID       string `json:"uuid"`
	Prefix     string `json:"prefix"`
	Fast       bool   `json:"fast"`
	ObjCached  bool   `json:"cached"`
	BckPresent bool   `json:"present"`
}

to generate bucket summary (or summaries)

type BsummResult added in v1.3.18 

type BsummResult struct {
	ObjCount struct {
		Present uint64 `json:"obj_count_present,string"`
		Remote  uint64 `json:"obj_count_remote,string"`
	}
	ObjSize struct {
		Min int64 `json:"obj_min_size"`
		Avg int64 `json:"obj_avg_size"`
		Max int64 `json:"obj_max_size"`
	}
	TotalSize struct {
		OnDisk      uint64 `json:"size_on_disk,string"`          // sum(dir sizes) aka "apparent size"
		PresentObjs uint64 `json:"size_all_present_objs,string"` // sum(cached object sizes)
		RemoteObjs  uint64 `json:"size_all_remote_objs,string"`  // sum(all object sizes in a remote bucket)
		Disks       uint64 `json:"total_disks_size,string"`
	}
	UsedPct      uint64 `json:"used_pct"`
	IsBckPresent bool   `json:"is_present"` // in BMD
}

"summarized" result for a given bucket

type CapacityInfo 

type CapacityInfo struct {
	Used    uint64  `json:"fs_used,string"`
	Total   uint64  `json:"fs_capacity,string"`
	PctUsed float64 `json:"pct_fs_used"`
}

sysinfo

type ClusterSysInfo 

type ClusterSysInfo struct {
	Proxy  map[string]*MemCPUInfo `json:"proxy"`
	Target map[string]*TSysInfo   `json:"target"`
}

sysinfo

type ClusterSysInfoRaw 

type ClusterSysInfoRaw struct {
	Proxy  cos.JSONRawMsgs `json:"proxy"`
	Target cos.JSONRawMsgs `json:"target"`
}

sysinfo

type CopyBckMsg 

type CopyBckMsg struct {
	Prepend string `json:"prepend"` // destination naming, as in: dest-obj-name = Prepend + source-obj-name
	Prefix  string `json:"prefix"`  // prefix to select matching _source_ objects or virtual directories
	DryRun  bool   `json:"dry_run"` // visit all source objects, don't make any modifications
	Force   bool   `json:"force"`   // force running in presence of "limited coexistence" conflict
}

copy & (offline) transform bucket to bucket

type JoinNodeResult 

type JoinNodeResult struct {
	DaemonID    string `json:"daemon_id"`
	RebalanceID string `json:"rebalance_id"`
}

type ListRange added in v1.3.18 

type ListRange struct {
	Template string   `json:"template"`
	ObjNames []string `json:"objnames"`
}

List of object names _or_ a template specifying { Prefix, Regex, and/or Range }

func (*ListRange) HasTemplate added in v1.3.18 

func (lrm *ListRange) HasTemplate() bool

func (*ListRange) IsList added in v1.3.18 

func (lrm *ListRange) IsList() bool

type LsoMsg 

type LsoMsg struct {
	UUID              string `json:"uuid"`               // ID to identify a single multi-page request
	Props             string `json:"props"`              // comma-delimited, e.g. "checksum,size,custom" (see GetProps* enum)
	TimeFormat        string `json:"time_format"`        // RFC822 is the default
	Prefix            string `json:"prefix"`             // return obj names starting with prefix (TODO: e.g. "A.tar/tutorials/")
	StartAfter        string `json:"start_after"`        // start listing after (AIS buckets only)
	ContinuationToken string `json:"continuation_token"` // => LsoResult.ContinuationToken => LsoMsg.ContinuationToken
	SID               string `json:"target"`             // selected target to solely execute backend.list-objects
	Flags             uint64 `json:"flags,string"`       // enum {LsObjCached, ...} - "LsoMsg flags" above
	PageSize          uint   `json:"pagesize"`           // max entries returned by list objects call
}

func (*LsoMsg) AddProps 

func (lsmsg *LsoMsg) AddProps(propNames ...string)

func (*LsoMsg) Clone 

func (lsmsg *LsoMsg) Clone() *LsoMsg

func (*LsoMsg) IsFlagSet 

func (lsmsg *LsoMsg) IsFlagSet(flags uint64) bool

func (*LsoMsg) PropsSet 

func (lsmsg *LsoMsg) PropsSet() (s cos.StrSet)

func (*LsoMsg) SetFlag 

func (lsmsg *LsoMsg) SetFlag(flag uint64)

LsoMsg flags enum: LsObjCached, ...

func (*LsoMsg) WantOnlyName added in v1.3.16 

func (lsmsg *LsoMsg) WantOnlyName() bool

NOTE: internal usage

func (*LsoMsg) WantOnlyRemoteProps 

func (lsmsg *LsoMsg) WantOnlyRemoteProps() bool

func (*LsoMsg) WantProp 

func (lsmsg *LsoMsg) WantProp(propName string) bool

WantProp returns true if msg request requires to return propName property.

type MemCPUInfo added in v1.3.16 

type MemCPUInfo struct {
	MemUsed    uint64      `json:"mem_used"`
	MemAvail   uint64      `json:"mem_avail"`
	PctMemUsed float64     `json:"pct_mem_used"`
	PctCPUUsed float64     `json:"pct_cpu_used"`
	LoadAvg    sys.LoadAvg `json:"load_avg"`
}

func GetMemCPU added in v1.3.16 

func GetMemCPU() MemCPUInfo

type MountpathList 

type MountpathList struct {
	Available []string `json:"available"`
	WaitingDD []string `json:"waiting_dd"`
	Disabled  []string `json:"disabled"`
}

MountpathList contains two lists:

  • Available - list of local mountpaths available to the storage target
  • WaitingDD - waiting for resilvering completion to be detached or disabled (moved to `Disabled`)
  • Disabled - list of disabled mountpaths, the mountpaths that generated IO errors followed by (FSHC) health check, etc.

type TCBMsg 

type TCBMsg struct {
	// NOTE: resulting object names will have this extension, if specified.
	// NOTE: if source bucket has two (or more) objects with the same base name but different extension,
	// specifying this field might cause unintended override.
	// TODO: this field might not be required when range/list transformation is supported.
	Ext cos.StrKVs `json:"ext"`

	Transform
	CopyBckMsg
}

copy & (offline) transform bucket to bucket

func (*TCBMsg) ToName 

func (msg *TCBMsg) ToName(name string) string

Replace extension and add suffix if provided.

func (*TCBMsg) Validate 

func (msg *TCBMsg) Validate(isEtl bool) (err error)

type TCObjsMsg added in v1.3.18 

type TCObjsMsg struct {
	ListRange
	TxnUUID string `json:"-"`
	TCBMsg
	ContinueOnError bool `json:"coer"` // ditto; TODO above
}

Multi-object copy & transform (see also: TCBMsg)

type TSysInfo 

type TSysInfo struct {
	MemCPUInfo
	CapacityInfo
}

sysinfo

type Transform added in v1.3.16 

type Transform struct {
	Name    string       `json:"id,omitempty"`
	Timeout cos.Duration `json:"request_timeout,omitempty"`
}

copy & (offline) transform bucket to bucket

type URLPath 

type URLPath struct {
	L []string
	S string
}

func (URLPath) Join 

func (u URLPath) Join(words ...string) string

type WritePolicy 

type WritePolicy string

write policy (enum and accessors) applies to both AIS metadata and data; bucket-configurable with global defaults via cluster config

func (WritePolicy) IsImmediate 

func (wp WritePolicy) IsImmediate() bool

func (WritePolicy) Validate 

func (wp WritePolicy) Validate() (err error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.