apc

package
v1.3.24 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 27, 2024 License: MIT Imports: 13 Imported by: 7

Documentation

Overview

Package apc: API control messages and constants

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package apc: API constant and control messages

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved.

Package apc: API control messages and constants

  • Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source
const (
	// object level
	AceGET     = AccessAttrs(1) << iota
	AceObjHEAD // permission to get object props
	AcePUT
	AceAPPEND
	AceObjDELETE
	AceObjMOVE
	AcePromote
	// TODO: reserving the only perm that must be checked on the target side (NIY)
	AceObjUpdate
	// bucket metadata
	AceBckHEAD   // get bucket props and ACL
	AceObjLIST   // list objects in a bucket
	AcePATCH     // set bucket props
	AceBckSetACL // set bucket permissions
	// cluster level
	AceListBuckets
	AceShowCluster
	AceCreateBucket
	AceDestroyBucket
	AceMoveBucket
	AceAdmin
	// note: must be the last one
	AceMax
)

ACL aka access permissions

View Source
const (
	// encompasses all ACEs, current and future
	AccessAll      = AccessAttrs(^uint64(0))
	AllowAllAccess = "su"

	// read-only and read-write access to bucket
	AccessRO             = AceGET | AceObjHEAD | AceListBuckets | AceBckHEAD | AceObjLIST
	AllowReadOnlyAccess  = "ro"
	AccessRW             = AccessRO | AcePUT | AceAPPEND | AceObjDELETE | AceObjMOVE
	AllowReadWriteAccess = "rw"

	AccessNone = AccessAttrs(0)
)

derived (convenience) constants

View Source
const (
	ActCreateBck   = "create-bck"  // NOTE: compare w/ ActAddRemoteBck below
	ActDestroyBck  = "destroy-bck" // destroy bucket data and metadata
	ActSetBprops   = "set-bprops"
	ActResetBprops = "reset-bprops"

	ActSummaryBck = "summary-bck"

	ActECEncode  = "ec-encode" // erasure code a bucket
	ActECGet     = "ec-get"    // read erasure coded objects
	ActECPut     = "ec-put"    // erasure code objects
	ActECRespond = "ec-resp"   // respond to other targets' EC requests

	ActCopyBck = "copy-bck"
	ActETLBck  = "etl-bck"

	ActETLInline = "etl-inline"

	ActDsort    = "dsort"
	ActDownload = "download"

	ActBlobDl = "blob-download"

	ActMakeNCopies = "make-n-copies"
	ActPutCopies   = "put-copies"

	ActRebalance = "rebalance"
	ActMoveBck   = "move-bck"

	ActResilver = "resilver"

	ActElection = "election"

	ActLRU          = "lru"
	ActStoreCleanup = "cleanup-store"

	ActEvictRemoteBck = "evict-remote-bck" // evict remote bucket's data
	ActInvalListCache = "inval-listobj-cache"
	ActList           = "list"
	ActLoadLomCache   = "load-lom-cache"
	ActNewPrimary     = "new-primary"
	ActPromote        = "promote"
	ActRenameObject   = "rename-obj"

	// cp (reverse)
	ActResetStats  = "reset-stats"
	ActResetConfig = "reset-config"
	ActSetConfig   = "set-config"

	ActRotateLogs = "rotate-logs"

	ActShutdownCluster = "shutdown" // see also: ActShutdownNode

	// multi-object (via `ListRange`)
	ActCopyObjects     = "copy-listrange"
	ActDeleteObjects   = "delete-listrange"
	ActETLObjects      = "etl-listrange"
	ActEvictObjects    = "evict-listrange"
	ActPrefetchObjects = "prefetch-listrange"
	ActArchive         = "archive" // see ArchiveMsg

	ActAttachRemAis = "attach"
	ActDetachRemAis = "detach"

	ActEnableBackend  = "enable-bend"
	ActDisableBackend = "disable-bend"

	// Node maintenance & cluster membership (see also ActRmNodeUnsafe below)
	ActStartMaintenance = "start-maintenance" // put into maintenance state
	ActStopMaintenance  = "stop-maintenance"  // cancel maintenance state
	ActShutdownNode     = "shutdown-node"     // shutdown node
	ActDecommissionNode = "decommission-node" // start rebalance and, when done, remove node from Smap

	ActDecommissionCluster = "decommission" // decommission all nodes in the cluster (cleanup system data)

	ActAdminJoinTarget = "admin-join-target"
	ActSelfJoinTarget  = "self-join-target"
	ActAdminJoinProxy  = "admin-join-proxy"
	ActSelfJoinProxy   = "self-join-proxy"
	ActKeepaliveUpdate = "keepalive-update"

	// IC
	ActSendOwnershipTbl  = "ic-send-own-tbl"
	ActListenToNotif     = "watch-xaction"
	ActMergeOwnershipTbl = "ic-merge-own-tbl"
	ActRegGlobalXaction  = "reg-global-xaction"
)

ActMsg.Action includes Xaction.Kind == ActMsg.Action (when the action is asynchronous)

View Source
const (
	ActAddRemoteBck   = "add-remote-bck"         // add to BMD existing remote bucket, usually on the fly
	ActRmNodeUnsafe   = "rm-unsafe"              // primary => the node to be removed
	ActStartGFN       = "start-gfn"              // get-from-neighbor
	ActStopGFN        = "stop-gfn"               // off
	ActCleanupMarkers = "cleanup-markers"        // part of the target joining sequence
	ActSelfRemove     = "self-initiated-removal" // e.g., when losing last mountpath
)

internal use

View Source
const (
	// Actions on mountpaths (/v1/daemon/mountpaths)
	ActMountpathAttach  = "attach-mp"
	ActMountpathEnable  = "enable-mp"
	ActMountpathDetach  = "detach-mp"
	ActMountpathDisable = "disable-mp"

	ActMountpathRescan = "rescan-mp"
	ActMountpathFSHC   = "fshc-mp"

	// Actions on xactions
	ActXactStop  = Stop
	ActXactStart = Start

	// auxiliary
	ActTransient = "transient" // transient - in-memory only
)
View Source
const (
	ActBegin  = "begin"
	ActCommit = "commit"
	ActAbort  = "abort"

	ActQuery = "query"
)

xaction begin-commit phases and related control

View Source
const (
	NodeMaintenance  = "maintenance"
	NodeDecommission = "decommission"
)
View Source
const (
	ActEcOpen  = "open-ec-streams"
	ActEcClose = "close-ec-streams"
)
View Source
const (
	CompressAlways = "always"
	CompressNever  = "never"
)

Compression enum

View Source
const (
	Proxy  = "proxy"
	Target = "target"
)
View Source
const (
	DeploymentK8s = "K8s"
	DeploymentDev = "dev"
)

deployment types

View Source
const (
	ReadHeaderTimeout    = 16 * time.Second
	EnvReadHeaderTimeout = "AIS_READ_HEADER_TIMEOUT"
)

in re: "Slowloris Attack"

View Source
const (
	DefaultTimeout = time.Duration(-1)
	LongTimeout    = time.Duration(-2)
)

timeouts for intra-cluster requests

View Source
const (

	// bucket inventory - an alternative way to list (very large) buckets
	HdrInventory = aisPrefix + "Bucket-Inventory" // must be present and must be "true" (or "y", "yes", "on" case-insensitive)
	HdrInvName   = aisPrefix + "Inv-Name"         // optional; name of the inventory (to override the system default)
	HdrInvID     = aisPrefix + "Inv-Id"           // optional; inventory ID (ditto)

	// GET via x-blob-download
	HdrBlobDownload = aisPrefix + "Blob-Download" // must be present and must be "true" (or "y", "yes", "on" case-insensitive)
	HdrBlobChunk    = aisPrefix + "Blob-Chunk"    // optional; e.g., 1mb, 2MIB, 3m, or 1234567 (bytes)
	HdrBlobWorkers  = aisPrefix + "Blob-Workers"  // optional; the default number of workers is dfltNumWorkers in xs/blob_download.go

	// Bucket props headers
	HdrBucketProps      = aisPrefix + "Bucket-Props"       // => cmn.Bprops
	HdrBucketSumm       = aisPrefix + "Bucket-Summ"        // => cmn.BsummResult (see also: QparamFltPresence)
	HdrBucketVerEnabled = aisPrefix + "Versioning-Enabled" // Enable/disable object versioning in a bucket.
	HdrBackendProvider  = aisPrefix + "Provider"           // ProviderAmazon et al. - see cmn/bck.go.

	// including BucketProps.Extra.AWS
	HdrS3Region   = aisPrefix + "Cloud_region"
	HdrS3Endpoint = aisPrefix + "Endpoint"
	HdrS3Profile  = aisPrefix + "Profile"

	// including BucketProps.Extra.HTTP
	HdrOrigURLBck = aisPrefix + "Original-Url"

	// remote AIS
	HdrRemAisUUID  = aisPrefix + "Remote-Ais-Uuid"
	HdrRemAisAlias = aisPrefix + "Remote-Ais-Alias"
	HdrRemAisURL   = aisPrefix + "Remote-Ais-Url"

	HdrRemoteOffline = aisPrefix + "Remote-Offline" // When accessing cached remote bucket with no backend connectivity.

	// Object props headers
	HdrObjCksumType = aisPrefix + "Checksum-Type"  // Checksum type, one of SupportedChecksums().
	HdrObjCksumVal  = aisPrefix + "Checksum-Value" // Checksum value.
	HdrObjAtime     = aisPrefix + "Atime"          // Object access time.
	HdrObjCustomMD  = aisPrefix + "Custom-Md"      // Object custom metadata.
	HdrObjVersion   = aisPrefix + "Version"        // Object version/generation - ais or cloud.

	// Append object header
	HdrAppendHandle = aisPrefix + "Append-Handle"

	// api.PutApndArchArgs message flags
	HdrPutApndArchFlags = aisPrefix + "Pine"

	// Query objects handle header
	HdrHandle = aisPrefix + "Query-Handle"

	// Reverse proxy header
	HdrNodeID = aisPrefix + "Node-Id"

	// uptimes, respectively
	HdrNodeUptime    = aisPrefix + "Node-Uptime"
	HdrClusterUptime = aisPrefix + "Cluster-Uptime"
)
View Source
const (
	HdrAuthorization         = "Authorization" // https://developer.mozilla.org/en-US/docs/Web/HTTP/Hdrs/Authorization
	AuthenticationTypeBearer = "Bearer"
)

AuthN consts

View Source
const (
	HdrCallerID        = aisPrefix + "Caller-Id" // Marker of intra-cluster request.
	HdrT2TPutterID     = aisPrefix + "Putter-Id" // DaemonID of the target that performs intra-cluster PUT
	HdrCallerName      = aisPrefix + "Caller-Name"
	HdrCallerIsPrimary = aisPrefix + "Caller-Is-Primary"
	HdrCallerSmapVer   = aisPrefix + "Caller-Smap-Ver"

	HdrXactionID = aisPrefix + "Xaction-Id"

	// intra-cluster streams
	HdrSessID   = aisPrefix + "Session-Id"
	HdrCompress = aisPrefix + "Compress" // LZ4

	// Promote(dir)
	HdrPromoteNamesHash = aisPrefix + "Promote-Names-Hash"
	HdrPromoteNamesNum  = aisPrefix + "Promote-Names-Num"

	// EC
	HdrActiveEC = aisPrefix + "Ec"
)

Internally used headers

View Source
const (
	LocationPropSepa = ":"
	LsPropsSepa      = ","
)
View Source
const (
	// Applies to objects from the buckets with remote backends (e.g., to optimize-out listing remotes)
	// See related Flt* enum
	LsObjCached = 1 << iota

	LsMissing // include missing main obj (with copy existing)

	LsDeleted // include obj-s marked for deletion (TODO: not implemented yet)

	LsArchDir // expand archives as directories

	LsNameOnly // return only object names and, spearately, statuses
	LsNameSize // same as above and size (minor speedup)

	// same as fltPresence == apc.Present (see query.go)
	LsBckPresent

	// LsDontHeadRemote is introduced primarily to support GCP buckets with
	// ACL policies that allow public _anonymous_ access.
	//
	// It appears that sometimes those policies do honor HEAD(bucket),
	// while other times they don't, failing the request with 401 or 403 status.
	// See also:
	// * https://cloud.google.com/storage/docs/access-control/making-data-public
	// * cmd/cli/cli/const.go for `dontHeadRemoteFlag`
	// * `QparamDontHeadRemote` (this package)
	LsDontHeadRemote

	// To list remote buckets without adding them to aistore
	// See also:
	// * cmd/cli/cli/const.go for `dontAddRemoteFlag`
	// * `QparamDontAddRemote` (this package)
	LsDontAddRemote

	// cache list-objects results and use this cache to speed-up
	UseListObjsCache

	// For remote buckets - list only remote props (aka `wantOnlyRemote`). When false,
	// the default that's being used is: `WantOnlyRemoteProps` - see below.
	// When true, the request gets executed in a pass-through fashion whereby a single ais target
	// simply forwards it to the associated remote backend and delivers the results as is to the
	// requesting proxy and, subsequently, to client.
	LsWantOnlyRemoteProps

	// List objects without recursion (POSIX-wise).
	// See related feature flag: feat.DontOptimizeVirtualDir
	LsNoRecursion

	// For remote metadata-capable buckets (ie., bck.HasVersioningMD() == true):
	// - check whether remote version exists,
	// and if it does:
	// - check whether remote version differs from its in-cluster copy
	LsVerChanged

	// Do not return virtual subdirectories.
	// Background:
	// Currently,    `list-objects(ais://BUCKET)` never returns virtual subdirectories - while,
	// for instance, `list-objects(aws://BUCKET)` MAY return the latter.
	// To prevent this from happening, specify LsNoDirs flag.
	LsNoDirs
)

LsoMsg flags

View Source
const (
	MaxPageSizeAIS   = 10000
	MaxPageSizeAWS   = 1000
	MaxPageSizeGCP   = 1000
	MaxPageSizeAzure = 5000
)

max page sizes see also: bprops Extra.AWS.MaxPageSize

View Source
const (
	// Status
	LocOK = iota
	LocMisplacedNode
	LocMisplacedMountpath
	LocIsCopy
	LocIsCopyMissingObj

	// LsoEntry Flags
	EntryIsCached   = 1 << (EntryStatusBits + 1)
	EntryInArch     = 1 << (EntryStatusBits + 2)
	EntryIsDir      = 1 << (EntryStatusBits + 3)
	EntryIsArchive  = 1 << (EntryStatusBits + 4)
	EntryVerChanged = 1 << (EntryStatusBits + 5) // see also: QparamLatestVer, et al.
	EntryVerRemoved = 1 << (EntryStatusBits + 6) // ditto
)
View Source
const (
	EntryStatusBits = 5                          // N bits
	EntryStatusMask = (1 << EntryStatusBits) - 1 // mask for N low bits
)

ObjEntry.Flags field

View Source
const (
	GetPropsName     = "name"
	GetPropsSize     = "size"
	GetPropsVersion  = "version"
	GetPropsChecksum = "checksum"
	GetPropsAtime    = "atime"
	GetPropsCached   = "cached"
	GetPropsStatus   = "status"
	GetPropsCopies   = "copies"
	GetPropsEC       = "ec"
	GetPropsCustom   = "custom"
	GetPropsLocation = "location" // advanced usage
)

LsoMsg and HEAD(object) enum (NOTE: compare with `cmn.ObjectProps`)

View Source
const (
	AIS   = "ais"
	AWS   = "aws"
	Azure = "azure"
	GCP   = "gcp"
	HT    = "ht"

	AllProviders = "ais, aws (s3://), gcp (gs://), azure (az://), ht://" // NOTE: must include all

	NsUUIDPrefix = '@' // BEWARE: used by on-disk layout
	NsNamePrefix = '#' // BEWARE: used by on-disk layout

	// consistent with rfc2396.txt "Uniform Resource Identifiers (URI): Generic Syntax"
	BckProviderSeparator = "://"

	// scheme://
	DefaultScheme = "https"
	GSScheme      = "gs"
	S3Scheme      = "s3"
	AZScheme      = "az"
	AISScheme     = "ais"
)

Backend Provider enum

View Source
const (
	ArchAppend = 1 << iota
	ArchAppendIfExist
)

api.PutApndArchArgs flags

View Source
const (
	QparamWhat = "what" // "smap" | "bmd" | "config" | "stats" | "xaction" ... (enum below)

	QparamProps = "props" // e.g. "checksum, size"|"atime, size"|"cached"|"bucket, size"| ...

	QparamUUID    = "uuid"     // xaction
	QparamJobID   = "jobid"    // job
	QparamETLName = "etl_name" // etl

	QparamRegex      = "regex"       // dsort: list regex
	QparamOnlyActive = "only_active" // dsort: list only active

	// remove existing custom keys and store new custom metadata
	// NOTE: making an s/_/-/ naming exception because of the namesake CLI usage
	QparamNewCustom = "set-new-custom"

	// Main bucket query params.
	QparamProvider  = "provider" // aka backend provider or, simply, backend
	QparamNamespace = "namespace"

	// e.g., usage: copy bucket
	QparamBckTo = "bck_to"

	// Do not add remote bucket to cluster's BMD e.g. when checking existence
	// via api.HeadBucket
	// By default, when existence of a remote buckets is confirmed the bucket's
	// metadata gets automatically (and transactionally) added to the cluster's BMD.
	// This query parameter can be used to override the default behavior.
	QparamDontAddRemote = "dont_add_remote_bck_md"

	// Add remote bucket to BMD _unconditionally_ and without executing HEAD request
	// (to check access and load the bucket's properties)
	// NOTE: usage is limited to setting up bucket properties with alternative
	// profile and/or endpoint
	// See also:
	// - `LsDontHeadRemote`
	// - docs/bucket.md
	// - docs/cli/aws_profile_endpoint.md
	QparamDontHeadRemote = "dont_head_remote_bck"

	// When evicting, keep remote bucket in BMD (i.e., evict data only)
	QparamKeepRemote = "keep_bck_md"

	// (api.GetBucketInfo)
	// NOTE: non-empty value indicates api.GetBucketInfo; "true" value further requires "with remote obj-s"
	QparamBinfoWithOrWithoutRemote = "bsumm_remote"

	// "presence" in a given cluster shall not be be confused with "existence" (possibly, remote).
	// See also:
	// - Flt* enum below
	// - ListObjsMsg flags, docs/providers.md (for terminology)
	QparamFltPresence = "presence"

	// APPEND(object) operation - QparamAppendType enum below
	QparamAppendType   = "append_type"
	QparamAppendHandle = "append_handle"

	// HTTP bucket support.
	QparamOrigURL = "original_url"

	// Get logs
	QparamLogSev  = "severity" // see { LogInfo, ...} enum
	QparamLogOff  = "offset"
	QparamAllLogs = "all"

	// The following 4 (four) QparamArch* parameters are all intended for usage with sharded datasets,
	// whereby the shards are (.tar, .tgz (or .tar.gz), .zip, and/or .tar.lz4) formatted objects.
	//
	// For the most recently updated list of supported serialization formats, please see cmn/archive package.
	//
	// "archpath" and "archmime", respectively, specify archived pathname and expected format (mime type)
	// of the containing shard; the latter is especially usable with non-standard shard name extensions;
	QparamArchpath = "archpath"
	QparamArchmime = "archmime"

	// In addition, the following two closely related parameters can be used to select multiple matching files
	// from a given shard.
	//
	// In particular, "archregx" specifies prefix, suffix, WebDataset key, _or_ general-purpose regular expression
	// that can be used to match archived filenames, and select possibly multiple files
	// (that will be then archived as a TAR and returned in one shot);
	QparamArchregx = "archregx"

	// "archmode", on the other hand, tells aistore whether to interpret "archregx" (above) as a
	// a general-purpose regular expression or, alternatively, use it for a simple and fast string comparison;
	// the latter is further formalized as `MatchMode` enum in the cmn/archive package,
	// with enumerated values including: "regexp", "prefix", "suffix", "substr", "wdskey".
	//
	// for example:
	// - given a shard containing (subdir/aaa.jpg, subdir/aaa.json, subdir/bbb.jpg, subdir/bbb.json, ...)
	//   and "wdskey" = "subdir/aaa", aistore will match and return (subdir/aaa.jpg, subdir/aaa.json).
	QparamArchmode = "archmode" // see `MatchMode` enum in cmn/archive/read

	// Skip loading existing object's metadata, in part to
	// compare its Checksum and update its existing Version (if exists).
	// Can be used to reduce PUT latency when:
	// - we massively write new content into a bucket, and/or
	// - we simply don't care.
	QparamSkipVC = "skip_vc"

	// force operation
	// used to overcome certain restrictions, e.g.:
	// - shutdown the primary and the entire cluster
	// - attach invalid mountpath
	QparamForce = "frc"

	// same as `Versioning.ValidateWarmGet` (cluster config and bucket props)
	// - usage: GET and (copy|transform) x (bucket|multi-object) operations
	// - implies remote backend
	QparamLatestVer = "latest-ver"

	// in addition to the latest-ver (above), also entails removing remotely
	// deleted objects
	QparamSync = "synchronize"

	// validate (ie., recompute and check) in-cluster object's checksums
	QparamValidateCksum = "validate-checksum"

	// when true, skip nlog.Error and friends
	// (to opt-out logging too many messages and/or benign warnings)
	QparamSilent = "sln"

	// (see api.AttachMountpath vs. LocalConfig.FSP)
	QparamMpathLabel = "mountpath_label"
)

see related "GET(what)" set of APIs: api/cluster and api/daemon

View Source
const (
	FltExists         = iota // (object | bucket) exists inside and/or outside cluster
	FltExistsNoProps         // same as above but no need to return props/info
	FltPresent               // bucket: is present | object: present and properly located
	FltPresentNoProps        // same as above but no need to return props/info
	FltPresentCluster        // objects: present anywhere/anyhow _in_ the cluster as: replica, ec-slices, misplaced
	FltExistsOutside         // not present - exists _outside_ cluster (NOTE: currently, only list-buckets)
)

QparamFltPresence enum.

Descibes both buckets and objects with respect to their existence/presence (or non-existence/non-presence) in AIS cluster.

"FltPresent*" refers to availability ("presence") in the cluster. For details, see the values and comments below.

Remote object or bucket that is currently not present can still be accessed with the very first access making it "present", etc.

View Source
const (
	AppendOp = "append"
	FlushOp  = "flush"
)

QparamAppendType enum.

View Source
const (
	QparamHealthReadiness = "readiness" // to be used by external watchdogs (e.g. K8s)
	QparamAskPrimary      = "apr"       // true: the caller is directing health request to primary
	QparamPrimaryReadyReb = "prr"       // true: check whether primary is ready to start rebalancing cluster
)

health

View Source
const (
	QparamProxyID          = "pid" // ID of the redirecting proxy.
	QparamPrimaryCandidate = "can" // ID of the candidate for the primary proxy.
	QparamPrepare          = "prp" // true: request belongs to the "prepare" phase of the primary proxy election
	QparamNonElectable     = "nel" // true: proxy is non-electable for the primary role
	QparamUnixTime         = "utm" // Unix time since 01/01/70 UTC (nanoseconds)
	QparamIsGFNRequest     = "gfn" // true if the request is a Get-From-Neighbor
	QparamRebStatus        = "rbs" // true: get detailed rebalancing status
	QparamRebData          = "rbd" // true: get EC rebalance data (pulling data if push way fails)
	QparamClusterInfo      = "cii" // true: /Health to return `cos.NodeStateInfo` including cluster metadata versions and state flags
	QparamOWT              = "owt" // object write transaction enum { OwtPut, ..., OwtGet* }

	QparamDontResilver = "dntres" // true: do not resilver data off of mountpaths that are being disabled/detached

	// dsort
	QparamTotalCompressedSize       = "tcs"
	QparamTotalInputShardsExtracted = "tise"
	QparamTotalUncompressedSize     = "tunc"

	// 2PC transactions - control plane
	QparamNetwTimeout  = "xnt" // [begin, start-commit] timeout
	QparamHostTimeout  = "xht" // [begin, txn-done] timeout
	QparamWaitMetasync = "xwm" // true: wait for metasync (used only when there's an alternative)

	// promote
	QparamConfirmFshare = "confirm-fshr" // confirm file share
	QparamActNoXact     = "act-no-xact"  // execute synchronously, i.e. without xaction

	// Notification target's node ID (usually, the node that initiates the operation).
	QparamNotifyMe = "nft"
)

Internal query params.

View Source
const (
	// cluster metadata
	WhatSmap = "smap"
	WhatBMD  = "bmd"

	// config
	WhatNodeConfig    = "config"         // query specific node for (cluster config + overrides, local config)
	WhatClusterConfig = "cluster_config" // as the name implies; identical (compressed, checksummed, versioned) copy on each node

	// configured backends
	WhatBackends = "backends"

	// stats and status
	WhatNodeStatsV322          = "stats"       // [ backward compatibility ]
	WhatNodeStatsAndStatusV322 = "status"      // [ ditto ]
	WhatNodeStats              = "node_stats"  // redundant
	WhatNodeStatsAndStatus     = "node_status" // current

	WhatDiskRWUtilCap = "disk" // read/write stats, disk utilization, capacity

	WhatMetricNames = "metrics"

	// assorted
	WhatMountpaths = "mountpaths"
	WhatRemoteAIS  = "remote"
	WhatSmapVote   = "smapvote"
	WhatSysInfo    = "sysinfo"
	WhatTargetIPs  = "target_ips" // comma-separated list of all target IPs (compare w/ GetWhatSnode)

	// log
	WhatLog = "log"

	// xactions
	WhatOneXactStatus   = "status"      // IC status by uuid (returns a single matching xaction or none)
	WhatAllXactStatus   = "status_all"  // ditto - all matching xactions
	WhatXactStats       = "getxstats"   // stats: xaction by uuid
	WhatQueryXactStats  = "qryxstats"   // stats: all matching xactions
	WhatAllRunningXacts = "running_all" // e.g. e.g.: put-copies[D-ViE6HEL_j] list[H96Y7bhR2s] ...

	// internal
	WhatSnode    = "snode"
	WhatICBundle = "ic_bundle"

	// tls
	WhatCertificate = "tls_certificate"
)

QparamWhat enum.

View Source
const (
	LogInfo = "info"
	LogWarn = "warning"
	LogErr  = "error"
)

QparamLogSev enum.

View Source
const (
	// l1
	Version = "v1"
	// l2
	Buckets   = "buckets"
	Objects   = "objects"
	EC        = "ec"
	Download  = "download"
	Daemon    = "daemon"
	Cluster   = "cluster"
	Tokens    = "tokens"
	Metasync  = "metasync"
	Health    = "health"
	Vote      = "vote"
	ObjStream = "objstream"
	MsgStream = "msgstream"
	Reverse   = "reverse"
	Xactions  = "xactions"
	S3        = "s3"
	Txn       = "txn"      // 2PC
	Notifs    = "notifs"   // intra-cluster notifications
	Users     = "users"    // AuthN
	Clusters  = "clusters" // AuthN
	Roles     = "roles"    // AuthN
	IC        = "ic"       // information center

	Voteres  = "result"
	VoteInit = "init"
	PriStop  = "primary-stopping"

	// (see the corresponding action messages above)
	Keepalive = "keepalive"
	AdminJoin = "join-by-admin" // when node is joined by admin ("manual join")
	SelfJoin  = "autoreg"       // auto-join cluster at startup

	// target
	Mountpaths = "mountpaths"

	// common
	Init     = "init"
	Start    = "start"
	Stop     = "stop"
	Abort    = "abort"
	Sort     = "sort"
	Finished = "finished"
	Progress = "progress"

	// dsort, dloader, query
	Metrics     = "metrics"
	Records     = "records"
	Shards      = "shards"
	FinishedAck = "finished_ack"
	UList       = "list"
	Remove      = "remove"
	Next        = "next"
	Peek        = "peek"
	Discard     = "discard"
	WorkerOwner = "worker" // TODO: it should be removed once get-next-bytes endpoint is ready

	LoadX509 = "load-x509"

	// ETL
	ETL        = "etl"
	ETLInfo    = "info"
	ETLList    = UList
	ETLLogs    = "logs"
	ETLObject  = "_object"
	ETLStop    = Stop
	ETLStart   = Start
	ETLHealth  = "health"
	ETLMetrics = "metrics"
)

RESTful URL path: l1/l2/l3

View Source
const (
	WriteImmediate = WritePolicy("immediate") // immediate write (default)
	WriteDelayed   = WritePolicy("delayed")   // cache and flush when not accessed for a while (lom_cache_hk.go)
	WriteNever     = WritePolicy("never")     // transient - in-memory only

	WriteDefault = WritePolicy("") // same as `WriteImmediate` - see IsImmediate() below
)
View Source
const GetPropsNameSize = GetPropsName + LsPropsSepa + GetPropsSize
View Source
const HdrError = "Hdr-Error"
View Source
const LZ4Compression = "lz4"

sent via req.Header.Set(apc.HdrCompress, LZ4Compression) (alternative to lz4 compressions upon popular request)

View Source
const NilValue = "none" // features (flags), log modules, et al.
View Source
const RemAIS = "remais" // to differentiate ais vs ais; also, default (remote ais cluster) alias
View Source
const (
	SyncSmap = "syncsmap"
)

RESTful l3, internal use

Variables

NOTE: update when changing any of the above :NOTE

View Source
var (
	URLPathS3 = urlpath(S3) // URLPath{[]string{S3}, S3}

	URLPathBuckets  = urlpath(Version, Buckets)
	URLPathObjects  = urlpath(Version, Objects)
	URLPathEC       = urlpath(Version, EC)
	URLPathNotifs   = urlpath(Version, Notifs)
	URLPathTxn      = urlpath(Version, Txn)
	URLPathXactions = urlpath(Version, Xactions)
	URLPathIC       = urlpath(Version, IC)
	URLPathHealth   = urlpath(Version, Health)
	URLPathMetasync = urlpath(Version, Metasync)

	URLPathClu        = urlpath(Version, Cluster)
	URLPathCluProxy   = urlpath(Version, Cluster, Proxy)
	URLPathCluUserReg = urlpath(Version, Cluster, AdminJoin)
	URLPathCluAutoReg = urlpath(Version, Cluster, SelfJoin)
	URLPathCluKalive  = urlpath(Version, Cluster, Keepalive)
	URLPathCluDaemon  = urlpath(Version, Cluster, Daemon) // (internal)
	URLPathCluSetConf = urlpath(Version, Cluster, ActSetConfig)
	URLPathCluAttach  = urlpath(Version, Cluster, ActAttachRemAis)
	URLPathCluDetach  = urlpath(Version, Cluster, ActDetachRemAis)

	URLPathCluX509 = urlpath(Version, Cluster, LoadX509)

	URLPathCluBendDisable = urlpath(Version, Cluster, ActDisableBackend)
	URLPathCluBendEnable  = urlpath(Version, Cluster, ActEnableBackend)

	URLPathDae          = urlpath(Version, Daemon)
	URLPathDaeProxy     = urlpath(Version, Daemon, Proxy)
	URLPathDaeSetConf   = urlpath(Version, Daemon, ActSetConfig)
	URLPathDaeAdminJoin = urlpath(Version, Daemon, AdminJoin)

	URLPathDaeBendDisable = urlpath(Version, Daemon, ActDisableBackend)
	URLPathDaeBendEnable  = urlpath(Version, Daemon, ActEnableBackend)

	URLPathDaeX509 = urlpath(Version, Daemon, LoadX509)

	URLPathReverse    = urlpath(Version, Reverse)
	URLPathReverseDae = urlpath(Version, Reverse, Daemon)

	URLPathVote        = urlpath(Version, Vote)
	URLPathVoteInit    = urlpath(Version, Vote, Init)
	URLPathVoteProxy   = urlpath(Version, Vote, Proxy)
	URLPathVoteVoteres = urlpath(Version, Vote, Voteres)
	URLPathVotePriStop = urlpath(Version, Vote, PriStop)

	URLPathdSort        = urlpath(Version, Sort)
	URLPathdSortInit    = urlpath(Version, Sort, Init)
	URLPathdSortStart   = urlpath(Version, Sort, Start)
	URLPathdSortList    = urlpath(Version, Sort, UList)
	URLPathdSortAbort   = urlpath(Version, Sort, Abort)
	URLPathdSortShards  = urlpath(Version, Sort, Shards)
	URLPathdSortRecords = urlpath(Version, Sort, Records)
	URLPathdSortMetrics = urlpath(Version, Sort, Metrics)
	URLPathdSortAck     = urlpath(Version, Sort, FinishedAck)
	URLPathdSortRemove  = urlpath(Version, Sort, Remove)

	URLPathDownload       = urlpath(Version, Download)
	URLPathDownloadAbort  = urlpath(Version, Download, Abort)
	URLPathDownloadRemove = urlpath(Version, Download, Remove)

	URLPathETL       = urlpath(Version, ETL)
	URLPathETLObject = urlpath(Version, ETL, ETLObject)

	URLPathTokens   = urlpath(Version, Tokens) // authn
	URLPathUsers    = urlpath(Version, Users)
	URLPathClusters = urlpath(Version, Clusters)
	URLPathRoles    = urlpath(Version, Roles)
)
View Source
var Providers = cos.NewStrSet(AIS, GCP, AWS, Azure, HT)
View Source
var SupportedCompression = [...]string{CompressNever, CompressAlways}

Functions

func AccessOp

func AccessOp(access AccessAttrs) string

func DisplayProvider

func DisplayProvider(p string) string

func IsCloudProvider added in v1.3.16

func IsCloudProvider(p string) bool

func IsFltNoProps

func IsFltNoProps(v int) bool

func IsFltPresent

func IsFltPresent(v int) bool

func IsProvider

func IsProvider(p string) bool

func IsRemoteProvider added in v1.3.16

func IsRemoteProvider(p string) bool

NOTE: not to confuse w/ bck.IsRemote() which also includes remote AIS

func IsValidCompression

func IsValidCompression(c string) bool

func NormalizeProvider

func NormalizeProvider(p string) string

func PropToHeader added in v1.3.22

func PropToHeader(prop string) string

internal (json) obj prop => canonical http header usage: - target InitObjProps2Hdr - api/object

func Ptr added in v1.3.22

func Ptr[T any](v T) *T

func SupportedPermissions

func SupportedPermissions() []string

verbs

func ToScheme

func ToScheme(p string) string

Types

type AccessAttrs

type AccessAttrs uint64

func StrToAccess

func StrToAccess(accessStr string) (access AccessAttrs, err error)

func (AccessAttrs) Describe

func (a AccessAttrs) Describe(all bool) string

func (AccessAttrs) Has

func (a AccessAttrs) Has(perms AccessAttrs) bool

func (AccessAttrs) String

func (a AccessAttrs) String() string

type ActMsg added in v1.3.16

type ActMsg struct {
	Value  any    `json:"value"`  // action-specific and optional
	Action string `json:"action"` // ActShutdown, ActRebalance, and many more (see apc/const.go)
	Name   string `json:"name"`   // action-specific info of any kind (not necessarily "name")
}

ActMsg is a JSON-formatted control structures used in a majority of API calls

func (*ActMsg) String added in v1.3.16

func (msg *ActMsg) String() string

func (*ActMsg) StringEx added in v1.3.18

func (msg *ActMsg) StringEx() string

type ActValRmNode

type ActValRmNode struct {
	DaemonID          string `json:"sid"`
	SkipRebalance     bool   `json:"skip_rebalance"`
	RmUserData        bool   `json:"rm_user_data"`        // decommission-only
	KeepInitialConfig bool   `json:"keep_initial_config"` // ditto (to be able to restart a node from scratch)
	NoShutdown        bool   `json:"no_shutdown"`
}

ActMsg is a JSON-formatted control structures used in a majority of API calls

type ArchiveMsg added in v1.3.18

type ArchiveMsg struct {
	TxnUUID     string `json:"-"`        // internal use
	FromBckName string `json:"-"`        // ditto
	ArchName    string `json:"archname"` // one of the archive.FileExtensions
	Mime        string `json:"mime"`     // user-specified mime type (NOTE: takes precedence if defined)
	ListRange
	BaseNameOnly    bool `json:"bnonly"` // only extract the base name of objects as names of archived objects
	InclSrcBname    bool `json:"isbn"`   // include source bucket name into the names of archived objects
	AppendIfExists  bool `json:"aate"`   // adding a list or a range of objects to an existing archive
	ContinueOnError bool `json:"coer"`   // on err, keep running arc xaction in a any given multi-object transaction
}

ArchiveMsg contains the parameters (all except the destination bucket) for archiving mutiple objects as one of the supported archive.FileExtensions types at the specified (bucket) destination. See also: api.PutApndArchArgs -------------------- terminology --------------------- here and elsewhere "archive" is any (.tar, .tgz/.tar.gz, .zip, .tar.lz4) formatted object. [NOTE] see cmn/api for cmn.ArchiveMsg (that also contains ToBck)

type BlobMsg added in v1.3.22

type BlobMsg struct {
	ChunkSize  int64 `json:"chunk-size"`  // as in: chunk size
	FullSize   int64 `json:"full-size"`   // user-specified (full) size of the object to download
	NumWorkers int   `json:"num-workers"` // number of concurrent downloading workers (readers); `dfltNumWorkers` when zero
	LatestVer  bool  `json:"latest-ver"`  // when true and in-cluster: check with remote whether (deleted | version-changed)
}

func (*BlobMsg) FromHeader added in v1.3.22

func (msg *BlobMsg) FromHeader(hdr http.Header) error

using textproto.CanonicalMIMEHeaderKey() to check presence - if a given key is present and is an empty string, it's an error

type BsummCtrlMsg added in v1.3.18

type BsummCtrlMsg struct {
	UUID          string `json:"uuid"`
	Prefix        string `json:"prefix"`
	ObjCached     bool   `json:"cached"`
	BckPresent    bool   `json:"present"`
	DontAddRemote bool   `json:"dont_add_remote"`
}

to generate bucket summary (or summaries)

type BsummResult added in v1.3.18

type BsummResult struct {
	ObjCount struct {
		Present uint64 `json:"obj_count_present,string"`
		Remote  uint64 `json:"obj_count_remote,string"`
	}
	ObjSize struct {
		Min int64 `json:"obj_min_size"`
		Avg int64 `json:"obj_avg_size"`
		Max int64 `json:"obj_max_size"`
	}
	TotalSize struct {
		OnDisk      uint64 `json:"size_on_disk,string"`          // sum(dir sizes) aka "apparent size"
		PresentObjs uint64 `json:"size_all_present_objs,string"` // sum(cached object sizes)
		RemoteObjs  uint64 `json:"size_all_remote_objs,string"`  // sum(all object sizes in a remote bucket)
		Disks       uint64 `json:"total_disks_size,string"`
	}
	UsedPct      uint64 `json:"used_pct"`
	IsBckPresent bool   `json:"is_present"` // in BMD
}

"summarized" result for a given bucket

type CapacityInfo

type CapacityInfo struct {
	Used    uint64  `json:"fs_used,string"`
	Total   uint64  `json:"fs_capacity,string"`
	PctUsed float64 `json:"pct_fs_used"`
}

sysinfo

type ClusterSysInfo

type ClusterSysInfo struct {
	Proxy  map[string]*MemCPUInfo `json:"proxy"`
	Target map[string]*TSysInfo   `json:"target"`
}

sysinfo

type ClusterSysInfoRaw

type ClusterSysInfoRaw struct {
	Proxy  cos.JSONRawMsgs `json:"proxy"`
	Target cos.JSONRawMsgs `json:"target"`
}

sysinfo

type CopyBckMsg

type CopyBckMsg struct {
	Prepend   string `json:"prepend"`     // destination naming, as in: dest-obj-name = Prepend + source-obj-name
	Prefix    string `json:"prefix"`      // prefix to select matching _source_ objects or virtual directories
	DryRun    bool   `json:"dry_run"`     // visit all source objects, don't make any modifications
	Force     bool   `json:"force"`       // force running in presence of "limited coexistence" type conflicts
	LatestVer bool   `json:"latest-ver"`  // see also: QparamLatestVer, 'versioning.validate_warm_get', PrefetchMsg
	Sync      bool   `json:"synchronize"` // see also: 'versioning.synchronize'
}

copy & (offline) transform bucket to bucket

type JoinNodeResult

type JoinNodeResult struct {
	DaemonID    string `json:"daemon_id"`
	RebalanceID string `json:"rebalance_id"`
}

type ListRange added in v1.3.18

type ListRange struct {
	Template string   `json:"template"`
	ObjNames []string `json:"objnames"`
}

List of object names _or_ a template specifying { optional Prefix, zero or more Ranges }

func (*ListRange) HasTemplate added in v1.3.18

func (lrm *ListRange) HasTemplate() bool

func (*ListRange) IsList added in v1.3.18

func (lrm *ListRange) IsList() bool

type LsoMsg

type LsoMsg struct {
	UUID              string      `json:"uuid"`                  // ID to identify a single multi-page request
	Props             string      `json:"props"`                 // comma-delimited, e.g. "checksum,size,custom" (see GetProps* enum)
	TimeFormat        string      `json:"time_format,omitempty"` // RFC822 is the default
	Prefix            string      `json:"prefix"`                // return obj names starting with prefix (TODO: e.g. "A.tar/tutorials/")
	StartAfter        string      `json:"start_after,omitempty"` // start listing after (AIS buckets only)
	ContinuationToken string      `json:"continuation_token"`    // => LsoResult.ContinuationToken => LsoMsg.ContinuationToken
	SID               string      `json:"target"`                // selected target to solely execute backend.list-objects
	Flags             uint64      `json:"flags,string"`          // enum {LsObjCached, ...} - "LsoMsg flags" above
	PageSize          int64       `json:"pagesize"`              // max entries returned by list objects call
	Header            http.Header `json:"hdr,omitempty"`         // (for pointers, see `ListArgs` in api/ls.go)
}

func (*LsoMsg) AddProps

func (lsmsg *LsoMsg) AddProps(propNames ...string)

func (*LsoMsg) ClearFlag added in v1.3.21

func (lsmsg *LsoMsg) ClearFlag(flag uint64)

func (*LsoMsg) Clone

func (lsmsg *LsoMsg) Clone() *LsoMsg

func (*LsoMsg) IsFlagSet

func (lsmsg *LsoMsg) IsFlagSet(flags uint64) bool

func (*LsoMsg) PropsSet

func (lsmsg *LsoMsg) PropsSet() (s cos.StrSet)

func (*LsoMsg) SetFlag

func (lsmsg *LsoMsg) SetFlag(flag uint64)

LsoMsg flags enum: LsObjCached, ...

func (*LsoMsg) WantOnlyRemoteProps

func (lsmsg *LsoMsg) WantOnlyRemoteProps() bool

func (*LsoMsg) WantProp

func (lsmsg *LsoMsg) WantProp(propName string) bool

WantProp returns true if msg request requires to return propName property.

type MemCPUInfo added in v1.3.16

type MemCPUInfo struct {
	MemUsed    uint64      `json:"mem_used"`
	MemAvail   uint64      `json:"mem_avail"`
	PctMemUsed float64     `json:"pct_mem_used"`
	PctCPUUsed float64     `json:"pct_cpu_used"`
	LoadAvg    sys.LoadAvg `json:"load_avg"`
}

func GetMemCPU added in v1.3.16

func GetMemCPU() MemCPUInfo

type MountpathList

type MountpathList struct {
	Available []string `json:"available"`
	WaitingDD []string `json:"waiting_dd"`
	Disabled  []string `json:"disabled"`
}

MountpathList contains two lists:

  • Available - list of local mountpaths available to the storage target
  • WaitingDD - waiting for resilvering completion to be detached or disabled (moved to `Disabled`)
  • Disabled - list of disabled mountpaths, the mountpaths that generated IO errors followed by (FSHC) health check, etc.

type PrefetchMsg added in v1.3.22

type PrefetchMsg struct {
	ListRange
	BlobThreshold   int64 `json:"blob-threshold"` // when greater than threshold prefetch using blob-downloader; otherwise cold GET
	NumWorkers      int   `json:"num-workers"`    // number of concurrent workers; 0 - number of mountpaths (default); (-1) none
	ContinueOnError bool  `json:"coer"`           // ignore non-critical errors, keep going
	LatestVer       bool  `json:"latest-ver"`     // when true & in-cluster: check with remote whether (deleted | version-changed)
}

prefetch

type PromoteArgs added in v1.3.22

type PromoteArgs struct {
	DaemonID  string `json:"tid,omitempty"` // target ID
	SrcFQN    string `json:"src,omitempty"` // source file or directory (must be absolute pathname)
	ObjName   string `json:"obj,omitempty"` // destination object name or prefix
	Recursive bool   `json:"rcr,omitempty"` // recursively promote nested dirs
	// once successfully promoted:
	OverwriteDst bool `json:"ovw,omitempty"` // overwrite destination
	DeleteSrc    bool `json:"dls,omitempty"` // remove source when (and after) successfully promoting
	// explicit request _not_ to treat the source as a potential file share
	// and _not_ to try to auto-detect if it is;
	// (auto-detection takes time, etc.)
	SrcIsNotFshare bool `json:"notshr,omitempty"` // the source is not a file share equally accessible by all targets
}

common part that's used in `api.PromoteArgs` and `PromoteParams`(server side), both

type TCBMsg

type TCBMsg struct {
	// NOTE: objname extension ----------------------------------------------------------------------
	// - resulting object names will have this extension, if specified.
	// - if source bucket has two (or more) objects with the same base name but different extension,
	//   specifying this field might cause unintended override.
	// - this field might not be any longer required - TODO review
	Ext cos.StrKVs `json:"ext"`

	Transform
	CopyBckMsg
}

copy & (offline) transform bucket to bucket

func (*TCBMsg) ToName

func (msg *TCBMsg) ToName(name string) string

Replace extension and add suffix if provided.

func (*TCBMsg) Validate

func (msg *TCBMsg) Validate(isEtl bool) (err error)

type TCOMsg added in v1.3.24

type TCOMsg struct {
	ListRange
	TxnUUID    string // (plstcx client, internal use)
	NumWorkers int    `json:"num-workers"` // number of concurrent workers; 0 - number of mountpaths (default); (-1) none
	TCBMsg
	ContinueOnError bool `json:"coer"`
}

multi-object copy & transform [NOTE] see cmn/api for cmn.TCOMsg (that also contains ToBck); see also TCBMsg

type TSysInfo

type TSysInfo struct {
	MemCPUInfo
	CapacityInfo
}

sysinfo

type Transform added in v1.3.16

type Transform struct {
	Name    string       `json:"id,omitempty"`
	Timeout cos.Duration `json:"request_timeout,omitempty"`
}

copy & (offline) transform bucket to bucket

type URLPath

type URLPath struct {
	L []string
	S string
}

func (URLPath) Join

func (u URLPath) Join(words ...string) string

type WritePolicy

type WritePolicy string

write policy (enum and accessors) applies to both AIS metadata and data; bucket-configurable with global defaults via cluster config

func (WritePolicy) IsImmediate

func (wp WritePolicy) IsImmediate() bool

func (WritePolicy) Validate

func (wp WritePolicy) Validate() (err error)

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL