Documentation ¶
Overview ¶
Package apc: API control messages and constants
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package apc: API constant and control messages
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved.
Package apc: API control messages and constants
- Copyright (c) 2018-2024, NVIDIA CORPORATION. All rights reserved.
Index ¶
- Constants
- Variables
- func AccessOp(access AccessAttrs) string
- func DisplayProvider(p string) string
- func IsCloudProvider(p string) bool
- func IsFltNoProps(v int) bool
- func IsFltPresent(v int) bool
- func IsProvider(p string) bool
- func IsRemoteProvider(p string) bool
- func IsValidCompression(c string) bool
- func NormalizeProvider(p string) string
- func PropToHeader(prop string) string
- func Ptr[T any](v T) *T
- func SupportedPermissions() []string
- func ToScheme(p string) string
- type AccessAttrs
- type ActMsg
- type ActValRmNode
- type ArchiveMsg
- type BlobMsg
- type BsummCtrlMsg
- type BsummResult
- type CapacityInfo
- type ClusterSysInfo
- type ClusterSysInfoRaw
- type CopyBckMsg
- type JoinNodeResult
- type ListRange
- type LsoMsg
- func (lsmsg *LsoMsg) AddProps(propNames ...string)
- func (lsmsg *LsoMsg) ClearFlag(flag uint64)
- func (lsmsg *LsoMsg) Clone() *LsoMsg
- func (lsmsg *LsoMsg) IsFlagSet(flags uint64) bool
- func (lsmsg *LsoMsg) PropsSet() (s cos.StrSet)
- func (lsmsg *LsoMsg) SetFlag(flag uint64)
- func (lsmsg *LsoMsg) WantOnlyRemoteProps() bool
- func (lsmsg *LsoMsg) WantProp(propName string) bool
- type MemCPUInfo
- type MountpathList
- type PrefetchMsg
- type PromoteArgs
- type TCBMsg
- type TCOMsg
- type TSysInfo
- type Transform
- type URLPath
- type WritePolicy
Constants ¶
const ( // object level AceGET = AccessAttrs(1) << iota AceObjHEAD // permission to get object props AcePUT AceAPPEND AceObjDELETE AceObjMOVE AcePromote // TODO: reserving the only perm that must be checked on the target side (NIY) AceObjUpdate // bucket metadata AceBckHEAD // get bucket props and ACL AceObjLIST // list objects in a bucket AcePATCH // set bucket props AceBckSetACL // set bucket permissions // cluster level AceListBuckets AceShowCluster AceCreateBucket AceDestroyBucket AceMoveBucket AceAdmin // note: must be the last one AceMax )
ACL aka access permissions
const ( // encompasses all ACEs, current and future AccessAll = AccessAttrs(^uint64(0)) AllowAllAccess = "su" // read-only and read-write access to bucket AccessRO = AceGET | AceObjHEAD | AceListBuckets | AceBckHEAD | AceObjLIST AllowReadOnlyAccess = "ro" AccessRW = AccessRO | AcePUT | AceAPPEND | AceObjDELETE | AceObjMOVE AllowReadWriteAccess = "rw" AccessNone = AccessAttrs(0) )
derived (convenience) constants
const ( ActCreateBck = "create-bck" // NOTE: compare w/ ActAddRemoteBck below ActDestroyBck = "destroy-bck" // destroy bucket data and metadata ActSetBprops = "set-bprops" ActResetBprops = "reset-bprops" ActSummaryBck = "summary-bck" ActECEncode = "ec-encode" // erasure code a bucket ActECGet = "ec-get" // read erasure coded objects ActECPut = "ec-put" // erasure code objects ActECRespond = "ec-resp" // respond to other targets' EC requests ActCopyBck = "copy-bck" ActETLBck = "etl-bck" ActETLInline = "etl-inline" ActDsort = "dsort" ActDownload = "download" ActBlobDl = "blob-download" ActMakeNCopies = "make-n-copies" ActPutCopies = "put-copies" ActRebalance = "rebalance" ActMoveBck = "move-bck" ActResilver = "resilver" ActElection = "election" ActLRU = "lru" ActStoreCleanup = "cleanup-store" ActEvictRemoteBck = "evict-remote-bck" // evict remote bucket's data ActInvalListCache = "inval-listobj-cache" ActList = "list" ActLoadLomCache = "load-lom-cache" ActNewPrimary = "new-primary" ActPromote = "promote" ActRenameObject = "rename-obj" // cp (reverse) ActResetStats = "reset-stats" ActResetConfig = "reset-config" ActSetConfig = "set-config" ActRotateLogs = "rotate-logs" ActShutdownCluster = "shutdown" // see also: ActShutdownNode // multi-object (via `ListRange`) ActCopyObjects = "copy-listrange" ActDeleteObjects = "delete-listrange" ActETLObjects = "etl-listrange" ActEvictObjects = "evict-listrange" ActPrefetchObjects = "prefetch-listrange" ActArchive = "archive" // see ArchiveMsg ActAttachRemAis = "attach" ActDetachRemAis = "detach" ActEnableBackend = "enable-bend" ActDisableBackend = "disable-bend" // Node maintenance & cluster membership (see also ActRmNodeUnsafe below) ActStartMaintenance = "start-maintenance" // put into maintenance state ActStopMaintenance = "stop-maintenance" // cancel maintenance state ActShutdownNode = "shutdown-node" // shutdown node ActDecommissionNode = "decommission-node" // start rebalance and, when done, remove node from Smap ActDecommissionCluster = "decommission" // decommission all nodes in the cluster (cleanup system data) ActAdminJoinTarget = "admin-join-target" ActSelfJoinTarget = "self-join-target" ActAdminJoinProxy = "admin-join-proxy" ActSelfJoinProxy = "self-join-proxy" ActKeepaliveUpdate = "keepalive-update" // IC ActSendOwnershipTbl = "ic-send-own-tbl" ActListenToNotif = "watch-xaction" ActMergeOwnershipTbl = "ic-merge-own-tbl" ActRegGlobalXaction = "reg-global-xaction" )
ActMsg.Action includes Xaction.Kind == ActMsg.Action (when the action is asynchronous)
const ( ActAddRemoteBck = "add-remote-bck" // add to BMD existing remote bucket, usually on the fly ActRmNodeUnsafe = "rm-unsafe" // primary => the node to be removed ActStartGFN = "start-gfn" // get-from-neighbor ActStopGFN = "stop-gfn" // off ActCleanupMarkers = "cleanup-markers" // part of the target joining sequence ActSelfRemove = "self-initiated-removal" // e.g., when losing last mountpath )
internal use
const ( // Actions on mountpaths (/v1/daemon/mountpaths) ActMountpathAttach = "attach-mp" ActMountpathEnable = "enable-mp" ActMountpathDetach = "detach-mp" ActMountpathDisable = "disable-mp" ActMountpathRescan = "rescan-mp" ActMountpathFSHC = "fshc-mp" // Actions on xactions ActXactStop = Stop ActXactStart = Start // auxiliary ActTransient = "transient" // transient - in-memory only )
const ( ActBegin = "begin" ActCommit = "commit" ActAbort = "abort" ActQuery = "query" )
xaction begin-commit phases and related control
const ( NodeMaintenance = "maintenance" NodeDecommission = "decommission" )
const ( ActEcOpen = "open-ec-streams" ActEcClose = "close-ec-streams" )
const ( CompressAlways = "always" CompressNever = "never" )
Compression enum
const ( Proxy = "proxy" Target = "target" )
const ( DeploymentK8s = "K8s" DeploymentDev = "dev" )
deployment types
const ( ReadHeaderTimeout = 16 * time.Second EnvReadHeaderTimeout = "AIS_READ_HEADER_TIMEOUT" )
in re: "Slowloris Attack"
const ( DefaultTimeout = time.Duration(-1) LongTimeout = time.Duration(-2) )
timeouts for intra-cluster requests
const ( // bucket inventory - an alternative way to list (very large) buckets HdrInventory = aisPrefix + "Bucket-Inventory" // must be present and must be "true" (or "y", "yes", "on" case-insensitive) HdrInvName = aisPrefix + "Inv-Name" // optional; name of the inventory (to override the system default) HdrInvID = aisPrefix + "Inv-Id" // optional; inventory ID (ditto) // GET via x-blob-download HdrBlobDownload = aisPrefix + "Blob-Download" // must be present and must be "true" (or "y", "yes", "on" case-insensitive) HdrBlobChunk = aisPrefix + "Blob-Chunk" // optional; e.g., 1mb, 2MIB, 3m, or 1234567 (bytes) HdrBlobWorkers = aisPrefix + "Blob-Workers" // optional; the default number of workers is dfltNumWorkers in xs/blob_download.go // Bucket props headers HdrBucketProps = aisPrefix + "Bucket-Props" // => cmn.Bprops HdrBucketSumm = aisPrefix + "Bucket-Summ" // => cmn.BsummResult (see also: QparamFltPresence) HdrBucketVerEnabled = aisPrefix + "Versioning-Enabled" // Enable/disable object versioning in a bucket. HdrBackendProvider = aisPrefix + "Provider" // ProviderAmazon et al. - see cmn/bck.go. // including BucketProps.Extra.AWS HdrS3Region = aisPrefix + "Cloud_region" HdrS3Endpoint = aisPrefix + "Endpoint" HdrS3Profile = aisPrefix + "Profile" // including BucketProps.Extra.HTTP HdrOrigURLBck = aisPrefix + "Original-Url" // remote AIS HdrRemAisUUID = aisPrefix + "Remote-Ais-Uuid" HdrRemAisAlias = aisPrefix + "Remote-Ais-Alias" HdrRemAisURL = aisPrefix + "Remote-Ais-Url" HdrRemoteOffline = aisPrefix + "Remote-Offline" // When accessing cached remote bucket with no backend connectivity. // Object props headers HdrObjCksumType = aisPrefix + "Checksum-Type" // Checksum type, one of SupportedChecksums(). HdrObjCksumVal = aisPrefix + "Checksum-Value" // Checksum value. HdrObjAtime = aisPrefix + "Atime" // Object access time. HdrObjCustomMD = aisPrefix + "Custom-Md" // Object custom metadata. HdrObjVersion = aisPrefix + "Version" // Object version/generation - ais or cloud. // Append object header HdrAppendHandle = aisPrefix + "Append-Handle" // api.PutApndArchArgs message flags HdrPutApndArchFlags = aisPrefix + "Pine" // Query objects handle header HdrHandle = aisPrefix + "Query-Handle" // Reverse proxy header HdrNodeID = aisPrefix + "Node-Id" // uptimes, respectively HdrNodeUptime = aisPrefix + "Node-Uptime" HdrClusterUptime = aisPrefix + "Cluster-Uptime" )
const ( HdrAuthorization = "Authorization" // https://developer.mozilla.org/en-US/docs/Web/HTTP/Hdrs/Authorization AuthenticationTypeBearer = "Bearer" )
AuthN consts
const ( HdrCallerID = aisPrefix + "Caller-Id" // Marker of intra-cluster request. HdrT2TPutterID = aisPrefix + "Putter-Id" // DaemonID of the target that performs intra-cluster PUT HdrCallerName = aisPrefix + "Caller-Name" HdrCallerIsPrimary = aisPrefix + "Caller-Is-Primary" HdrCallerSmapVer = aisPrefix + "Caller-Smap-Ver" HdrXactionID = aisPrefix + "Xaction-Id" // intra-cluster streams HdrSessID = aisPrefix + "Session-Id" HdrCompress = aisPrefix + "Compress" // LZ4 // Promote(dir) HdrPromoteNamesHash = aisPrefix + "Promote-Names-Hash" HdrPromoteNamesNum = aisPrefix + "Promote-Names-Num" // EC HdrActiveEC = aisPrefix + "Ec" )
Internally used headers
const ( LocationPropSepa = ":" LsPropsSepa = "," )
const ( // Applies to objects from the buckets with remote backends (e.g., to optimize-out listing remotes) // See related Flt* enum LsObjCached = 1 << iota LsMissing // include missing main obj (with copy existing) LsDeleted // include obj-s marked for deletion (TODO: not implemented yet) LsArchDir // expand archives as directories LsNameOnly // return only object names and, spearately, statuses LsNameSize // same as above and size (minor speedup) // same as fltPresence == apc.Present (see query.go) LsBckPresent // LsDontHeadRemote is introduced primarily to support GCP buckets with // ACL policies that allow public _anonymous_ access. // // It appears that sometimes those policies do honor HEAD(bucket), // while other times they don't, failing the request with 401 or 403 status. // See also: // * https://cloud.google.com/storage/docs/access-control/making-data-public // * cmd/cli/cli/const.go for `dontHeadRemoteFlag` // * `QparamDontHeadRemote` (this package) LsDontHeadRemote // To list remote buckets without adding them to aistore // See also: // * cmd/cli/cli/const.go for `dontAddRemoteFlag` // * `QparamDontAddRemote` (this package) LsDontAddRemote // cache list-objects results and use this cache to speed-up UseListObjsCache // For remote buckets - list only remote props (aka `wantOnlyRemote`). When false, // the default that's being used is: `WantOnlyRemoteProps` - see below. // When true, the request gets executed in a pass-through fashion whereby a single ais target // simply forwards it to the associated remote backend and delivers the results as is to the // requesting proxy and, subsequently, to client. LsWantOnlyRemoteProps // List objects without recursion (POSIX-wise). // See related feature flag: feat.DontOptimizeVirtualDir LsNoRecursion // For remote metadata-capable buckets (ie., bck.HasVersioningMD() == true): // - check whether remote version exists, // and if it does: // - check whether remote version differs from its in-cluster copy LsVerChanged // Do not return virtual subdirectories. // Background: // Currently, `list-objects(ais://BUCKET)` never returns virtual subdirectories - while, // for instance, `list-objects(aws://BUCKET)` MAY return the latter. // To prevent this from happening, specify LsNoDirs flag. LsNoDirs )
LsoMsg flags
const ( MaxPageSizeAIS = 10000 MaxPageSizeAWS = 1000 MaxPageSizeGCP = 1000 MaxPageSizeAzure = 5000 )
max page sizes see also: bprops Extra.AWS.MaxPageSize
const ( // Status LocOK = iota LocMisplacedNode LocMisplacedMountpath LocIsCopy LocIsCopyMissingObj // LsoEntry Flags EntryIsCached = 1 << (EntryStatusBits + 1) EntryInArch = 1 << (EntryStatusBits + 2) EntryIsDir = 1 << (EntryStatusBits + 3) EntryIsArchive = 1 << (EntryStatusBits + 4) EntryVerChanged = 1 << (EntryStatusBits + 5) // see also: QparamLatestVer, et al. EntryVerRemoved = 1 << (EntryStatusBits + 6) // ditto )
const ( EntryStatusBits = 5 // N bits EntryStatusMask = (1 << EntryStatusBits) - 1 // mask for N low bits )
ObjEntry.Flags field
const ( GetPropsName = "name" GetPropsSize = "size" GetPropsVersion = "version" GetPropsChecksum = "checksum" GetPropsAtime = "atime" GetPropsCached = "cached" GetPropsStatus = "status" GetPropsCopies = "copies" GetPropsEC = "ec" GetPropsCustom = "custom" GetPropsLocation = "location" // advanced usage )
LsoMsg and HEAD(object) enum (NOTE: compare with `cmn.ObjectProps`)
const ( AIS = "ais" AWS = "aws" Azure = "azure" GCP = "gcp" HT = "ht" AllProviders = "ais, aws (s3://), gcp (gs://), azure (az://), ht://" // NOTE: must include all NsUUIDPrefix = '@' // BEWARE: used by on-disk layout NsNamePrefix = '#' // BEWARE: used by on-disk layout // consistent with rfc2396.txt "Uniform Resource Identifiers (URI): Generic Syntax" BckProviderSeparator = "://" // scheme:// DefaultScheme = "https" GSScheme = "gs" S3Scheme = "s3" AZScheme = "az" AISScheme = "ais" )
Backend Provider enum
const ( ArchAppend = 1 << iota ArchAppendIfExist )
api.PutApndArchArgs flags
const ( QparamWhat = "what" // "smap" | "bmd" | "config" | "stats" | "xaction" ... (enum below) QparamProps = "props" // e.g. "checksum, size"|"atime, size"|"cached"|"bucket, size"| ... QparamUUID = "uuid" // xaction QparamJobID = "jobid" // job QparamETLName = "etl_name" // etl QparamRegex = "regex" // dsort: list regex QparamOnlyActive = "only_active" // dsort: list only active // remove existing custom keys and store new custom metadata // NOTE: making an s/_/-/ naming exception because of the namesake CLI usage QparamNewCustom = "set-new-custom" // Main bucket query params. QparamProvider = "provider" // aka backend provider or, simply, backend QparamNamespace = "namespace" // e.g., usage: copy bucket QparamBckTo = "bck_to" // Do not add remote bucket to cluster's BMD e.g. when checking existence // via api.HeadBucket // By default, when existence of a remote buckets is confirmed the bucket's // metadata gets automatically (and transactionally) added to the cluster's BMD. // This query parameter can be used to override the default behavior. QparamDontAddRemote = "dont_add_remote_bck_md" // Add remote bucket to BMD _unconditionally_ and without executing HEAD request // (to check access and load the bucket's properties) // NOTE: usage is limited to setting up bucket properties with alternative // profile and/or endpoint // See also: // - `LsDontHeadRemote` // - docs/bucket.md // - docs/cli/aws_profile_endpoint.md QparamDontHeadRemote = "dont_head_remote_bck" // When evicting, keep remote bucket in BMD (i.e., evict data only) QparamKeepRemote = "keep_bck_md" // (api.GetBucketInfo) // NOTE: non-empty value indicates api.GetBucketInfo; "true" value further requires "with remote obj-s" QparamBinfoWithOrWithoutRemote = "bsumm_remote" // "presence" in a given cluster shall not be be confused with "existence" (possibly, remote). // See also: // - Flt* enum below // - ListObjsMsg flags, docs/providers.md (for terminology) QparamFltPresence = "presence" // APPEND(object) operation - QparamAppendType enum below QparamAppendType = "append_type" QparamAppendHandle = "append_handle" // HTTP bucket support. QparamOrigURL = "original_url" // Get logs QparamLogSev = "severity" // see { LogInfo, ...} enum QparamLogOff = "offset" QparamAllLogs = "all" // The following 4 (four) QparamArch* parameters are all intended for usage with sharded datasets, // whereby the shards are (.tar, .tgz (or .tar.gz), .zip, and/or .tar.lz4) formatted objects. // // For the most recently updated list of supported serialization formats, please see cmn/archive package. // // "archpath" and "archmime", respectively, specify archived pathname and expected format (mime type) // of the containing shard; the latter is especially usable with non-standard shard name extensions; QparamArchpath = "archpath" QparamArchmime = "archmime" // In addition, the following two closely related parameters can be used to select multiple matching files // from a given shard. // // In particular, "archregx" specifies prefix, suffix, WebDataset key, _or_ general-purpose regular expression // that can be used to match archived filenames, and select possibly multiple files // (that will be then archived as a TAR and returned in one shot); QparamArchregx = "archregx" // "archmode", on the other hand, tells aistore whether to interpret "archregx" (above) as a // a general-purpose regular expression or, alternatively, use it for a simple and fast string comparison; // the latter is further formalized as `MatchMode` enum in the cmn/archive package, // with enumerated values including: "regexp", "prefix", "suffix", "substr", "wdskey". // // for example: // - given a shard containing (subdir/aaa.jpg, subdir/aaa.json, subdir/bbb.jpg, subdir/bbb.json, ...) // and "wdskey" = "subdir/aaa", aistore will match and return (subdir/aaa.jpg, subdir/aaa.json). QparamArchmode = "archmode" // see `MatchMode` enum in cmn/archive/read // Skip loading existing object's metadata, in part to // compare its Checksum and update its existing Version (if exists). // Can be used to reduce PUT latency when: // - we massively write new content into a bucket, and/or // - we simply don't care. QparamSkipVC = "skip_vc" // force operation // used to overcome certain restrictions, e.g.: // - shutdown the primary and the entire cluster // - attach invalid mountpath QparamForce = "frc" // same as `Versioning.ValidateWarmGet` (cluster config and bucket props) // - usage: GET and (copy|transform) x (bucket|multi-object) operations // - implies remote backend QparamLatestVer = "latest-ver" // in addition to the latest-ver (above), also entails removing remotely // deleted objects QparamSync = "synchronize" // validate (ie., recompute and check) in-cluster object's checksums QparamValidateCksum = "validate-checksum" // when true, skip nlog.Error and friends // (to opt-out logging too many messages and/or benign warnings) QparamSilent = "sln" // (see api.AttachMountpath vs. LocalConfig.FSP) QparamMpathLabel = "mountpath_label" )
see related "GET(what)" set of APIs: api/cluster and api/daemon
const ( FltExists = iota // (object | bucket) exists inside and/or outside cluster FltExistsNoProps // same as above but no need to return props/info FltPresent // bucket: is present | object: present and properly located FltPresentNoProps // same as above but no need to return props/info FltPresentCluster // objects: present anywhere/anyhow _in_ the cluster as: replica, ec-slices, misplaced FltExistsOutside // not present - exists _outside_ cluster (NOTE: currently, only list-buckets) )
QparamFltPresence enum.
Descibes both buckets and objects with respect to their existence/presence (or non-existence/non-presence) in AIS cluster.
"FltPresent*" refers to availability ("presence") in the cluster. For details, see the values and comments below.
Remote object or bucket that is currently not present can still be accessed with the very first access making it "present", etc.
const ( AppendOp = "append" FlushOp = "flush" )
QparamAppendType enum.
const ( QparamHealthReadiness = "readiness" // to be used by external watchdogs (e.g. K8s) QparamAskPrimary = "apr" // true: the caller is directing health request to primary QparamPrimaryReadyReb = "prr" // true: check whether primary is ready to start rebalancing cluster )
health
const ( QparamProxyID = "pid" // ID of the redirecting proxy. QparamPrimaryCandidate = "can" // ID of the candidate for the primary proxy. QparamPrepare = "prp" // true: request belongs to the "prepare" phase of the primary proxy election QparamNonElectable = "nel" // true: proxy is non-electable for the primary role QparamUnixTime = "utm" // Unix time since 01/01/70 UTC (nanoseconds) QparamIsGFNRequest = "gfn" // true if the request is a Get-From-Neighbor QparamRebStatus = "rbs" // true: get detailed rebalancing status QparamRebData = "rbd" // true: get EC rebalance data (pulling data if push way fails) QparamClusterInfo = "cii" // true: /Health to return `cos.NodeStateInfo` including cluster metadata versions and state flags QparamOWT = "owt" // object write transaction enum { OwtPut, ..., OwtGet* } QparamDontResilver = "dntres" // true: do not resilver data off of mountpaths that are being disabled/detached // dsort QparamTotalCompressedSize = "tcs" QparamTotalInputShardsExtracted = "tise" QparamTotalUncompressedSize = "tunc" // 2PC transactions - control plane QparamNetwTimeout = "xnt" // [begin, start-commit] timeout QparamHostTimeout = "xht" // [begin, txn-done] timeout QparamWaitMetasync = "xwm" // true: wait for metasync (used only when there's an alternative) // promote QparamActNoXact = "act-no-xact" // execute synchronously, i.e. without xaction // Notification target's node ID (usually, the node that initiates the operation). QparamNotifyMe = "nft" )
Internal query params.
const ( // cluster metadata WhatSmap = "smap" WhatBMD = "bmd" // config WhatNodeConfig = "config" // query specific node for (cluster config + overrides, local config) WhatClusterConfig = "cluster_config" // as the name implies; identical (compressed, checksummed, versioned) copy on each node // configured backends WhatBackends = "backends" // stats and status WhatNodeStatsV322 = "stats" // [ backward compatibility ] WhatNodeStatsAndStatusV322 = "status" // [ ditto ] WhatNodeStats = "node_stats" // redundant WhatNodeStatsAndStatus = "node_status" // current WhatDiskRWUtilCap = "disk" // read/write stats, disk utilization, capacity WhatMetricNames = "metrics" // assorted WhatMountpaths = "mountpaths" WhatRemoteAIS = "remote" WhatSmapVote = "smapvote" WhatSysInfo = "sysinfo" WhatTargetIPs = "target_ips" // comma-separated list of all target IPs (compare w/ GetWhatSnode) // log WhatLog = "log" // xactions WhatOneXactStatus = "status" // IC status by uuid (returns a single matching xaction or none) WhatAllXactStatus = "status_all" // ditto - all matching xactions WhatXactStats = "getxstats" // stats: xaction by uuid WhatQueryXactStats = "qryxstats" // stats: all matching xactions WhatAllRunningXacts = "running_all" // e.g. e.g.: put-copies[D-ViE6HEL_j] list[H96Y7bhR2s] ... // internal WhatSnode = "snode" WhatICBundle = "ic_bundle" // tls WhatCertificate = "tls_certificate" )
QparamWhat enum.
const ( LogInfo = "info" LogWarn = "warning" LogErr = "error" )
QparamLogSev enum.
const ( // l1 Version = "v1" // l2 Buckets = "buckets" Objects = "objects" EC = "ec" Download = "download" Daemon = "daemon" Cluster = "cluster" Tokens = "tokens" Metasync = "metasync" Health = "health" Vote = "vote" ObjStream = "objstream" MsgStream = "msgstream" Reverse = "reverse" Xactions = "xactions" S3 = "s3" Txn = "txn" // 2PC Notifs = "notifs" // intra-cluster notifications Users = "users" // AuthN Clusters = "clusters" // AuthN Roles = "roles" // AuthN IC = "ic" // information center Voteres = "result" VoteInit = "init" PriStop = "primary-stopping" // (see the corresponding action messages above) Keepalive = "keepalive" AdminJoin = "join-by-admin" // when node is joined by admin ("manual join") SelfJoin = "autoreg" // auto-join cluster at startup // target Mountpaths = "mountpaths" // common Init = "init" Start = "start" Stop = "stop" Abort = "abort" Sort = "sort" Finished = "finished" Progress = "progress" // dsort, dloader, query Metrics = "metrics" Records = "records" Shards = "shards" FinishedAck = "finished_ack" UList = "list" Remove = "remove" Next = "next" Peek = "peek" Discard = "discard" WorkerOwner = "worker" // TODO: it should be removed once get-next-bytes endpoint is ready LoadX509 = "load-x509" // ETL ETL = "etl" ETLInfo = "info" ETLList = UList ETLLogs = "logs" ETLObject = "_object" ETLStop = Stop ETLStart = Start ETLHealth = "health" ETLMetrics = "metrics" )
RESTful URL path: l1/l2/l3
const ( WriteImmediate = WritePolicy("immediate") // immediate write (default) WriteDelayed = WritePolicy("delayed") // cache and flush when not accessed for a while (lom_cache_hk.go) WriteNever = WritePolicy("never") // transient - in-memory only WriteDefault = WritePolicy("") // same as `WriteImmediate` - see IsImmediate() below )
const GetPropsNameSize = GetPropsName + LsPropsSepa + GetPropsSize
const HdrError = "Hdr-Error"
const LZ4Compression = "lz4"
sent via req.Header.Set(apc.HdrCompress, LZ4Compression) (alternative to lz4 compressions upon popular request)
const NilValue = "none" // features (flags), log modules, et al.
const RemAIS = "remais" // to differentiate ais vs ais; also, default (remote ais cluster) alias
const (
SyncSmap = "syncsmap"
)
RESTful l3, internal use
Variables ¶
var ( GetPropsMinimal = []string{GetPropsName, GetPropsSize, GetPropsCached} GetPropsDefaultCloud = []string{GetPropsName, GetPropsSize, GetPropsCached, GetPropsChecksum, GetPropsVersion, GetPropsCustom} GetPropsDefaultAIS = []string{GetPropsName, GetPropsSize, GetPropsChecksum, GetPropsAtime} GetPropsAll = []string{GetPropsName, GetPropsSize, GetPropsChecksum, GetPropsAtime, GetPropsVersion, GetPropsCached, GetPropsStatus, GetPropsCopies, GetPropsEC, GetPropsCustom, GetPropsLocation} )
NOTE: update when changing any of the above :NOTE
var ( URLPathS3 = urlpath(S3) // URLPath{[]string{S3}, S3} URLPathBuckets = urlpath(Version, Buckets) URLPathObjects = urlpath(Version, Objects) URLPathEC = urlpath(Version, EC) URLPathNotifs = urlpath(Version, Notifs) URLPathTxn = urlpath(Version, Txn) URLPathXactions = urlpath(Version, Xactions) URLPathIC = urlpath(Version, IC) URLPathHealth = urlpath(Version, Health) URLPathMetasync = urlpath(Version, Metasync) URLPathClu = urlpath(Version, Cluster) URLPathCluProxy = urlpath(Version, Cluster, Proxy) URLPathCluUserReg = urlpath(Version, Cluster, AdminJoin) URLPathCluAutoReg = urlpath(Version, Cluster, SelfJoin) URLPathCluKalive = urlpath(Version, Cluster, Keepalive) URLPathCluDaemon = urlpath(Version, Cluster, Daemon) // (internal) URLPathCluSetConf = urlpath(Version, Cluster, ActSetConfig) URLPathCluAttach = urlpath(Version, Cluster, ActAttachRemAis) URLPathCluDetach = urlpath(Version, Cluster, ActDetachRemAis) URLPathCluX509 = urlpath(Version, Cluster, LoadX509) URLPathCluBendDisable = urlpath(Version, Cluster, ActDisableBackend) URLPathCluBendEnable = urlpath(Version, Cluster, ActEnableBackend) URLPathDae = urlpath(Version, Daemon) URLPathDaeProxy = urlpath(Version, Daemon, Proxy) URLPathDaeSetConf = urlpath(Version, Daemon, ActSetConfig) URLPathDaeAdminJoin = urlpath(Version, Daemon, AdminJoin) URLPathDaeBendDisable = urlpath(Version, Daemon, ActDisableBackend) URLPathDaeBendEnable = urlpath(Version, Daemon, ActEnableBackend) URLPathDaeX509 = urlpath(Version, Daemon, LoadX509) URLPathReverse = urlpath(Version, Reverse) URLPathReverseDae = urlpath(Version, Reverse, Daemon) URLPathVote = urlpath(Version, Vote) URLPathVoteInit = urlpath(Version, Vote, Init) URLPathVoteProxy = urlpath(Version, Vote, Proxy) URLPathVoteVoteres = urlpath(Version, Vote, Voteres) URLPathVotePriStop = urlpath(Version, Vote, PriStop) URLPathdSort = urlpath(Version, Sort) URLPathdSortInit = urlpath(Version, Sort, Init) URLPathdSortStart = urlpath(Version, Sort, Start) URLPathdSortList = urlpath(Version, Sort, UList) URLPathdSortAbort = urlpath(Version, Sort, Abort) URLPathdSortShards = urlpath(Version, Sort, Shards) URLPathdSortRecords = urlpath(Version, Sort, Records) URLPathdSortMetrics = urlpath(Version, Sort, Metrics) URLPathdSortAck = urlpath(Version, Sort, FinishedAck) URLPathdSortRemove = urlpath(Version, Sort, Remove) URLPathDownload = urlpath(Version, Download) URLPathDownloadAbort = urlpath(Version, Download, Abort) URLPathDownloadRemove = urlpath(Version, Download, Remove) URLPathETL = urlpath(Version, ETL) URLPathETLObject = urlpath(Version, ETL, ETLObject) URLPathTokens = urlpath(Version, Tokens) // authn URLPathUsers = urlpath(Version, Users) URLPathClusters = urlpath(Version, Clusters) URLPathRoles = urlpath(Version, Roles) )
var SupportedCompression = [...]string{CompressNever, CompressAlways}
var SupportedWritePolicy = [...]string{string(WriteImmediate), string(WriteDelayed), string(WriteNever)}
Functions ¶
func AccessOp ¶
func AccessOp(access AccessAttrs) string
func DisplayProvider ¶
func IsCloudProvider ¶ added in v1.3.16
func IsFltNoProps ¶
func IsFltPresent ¶
func IsProvider ¶
func IsRemoteProvider ¶ added in v1.3.16
NOTE: not to confuse w/ bck.IsRemote() which also includes remote AIS
func IsValidCompression ¶
func NormalizeProvider ¶
func PropToHeader ¶ added in v1.3.22
internal (json) obj prop => canonical http header usage: - target InitObjProps2Hdr - api/object
Types ¶
type AccessAttrs ¶
type AccessAttrs uint64
func StrToAccess ¶
func StrToAccess(accessStr string) (access AccessAttrs, err error)
func (AccessAttrs) Describe ¶
func (a AccessAttrs) Describe(all bool) string
func (AccessAttrs) Has ¶
func (a AccessAttrs) Has(perms AccessAttrs) bool
func (AccessAttrs) String ¶
func (a AccessAttrs) String() string
type ActMsg ¶ added in v1.3.16
type ActMsg struct { Value any `json:"value"` // action-specific and optional Action string `json:"action"` // ActShutdown, ActRebalance, and many more (see apc/const.go) Name string `json:"name"` // action-specific info of any kind (not necessarily "name") }
ActMsg is a JSON-formatted control structures used in a majority of API calls
type ActValRmNode ¶
type ActValRmNode struct { DaemonID string `json:"sid"` SkipRebalance bool `json:"skip_rebalance"` RmUserData bool `json:"rm_user_data"` // decommission-only KeepInitialConfig bool `json:"keep_initial_config"` // ditto (to be able to restart a node from scratch) NoShutdown bool `json:"no_shutdown"` }
ActMsg is a JSON-formatted control structures used in a majority of API calls
type ArchiveMsg ¶ added in v1.3.18
type ArchiveMsg struct { TxnUUID string `json:"-"` // internal use FromBckName string `json:"-"` // ditto ArchName string `json:"archname"` // one of the archive.FileExtensions Mime string `json:"mime"` // user-specified mime type (NOTE: takes precedence if defined) ListRange BaseNameOnly bool `json:"bnonly"` // only extract the base name of objects as names of archived objects InclSrcBname bool `json:"isbn"` // include source bucket name into the names of archived objects AppendIfExists bool `json:"aate"` // adding a list or a range of objects to an existing archive ContinueOnError bool `json:"coer"` // on err, keep running arc xaction in a any given multi-object transaction }
ArchiveMsg contains the parameters (all except the destination bucket) for archiving mutiple objects as one of the supported archive.FileExtensions types at the specified (bucket) destination. See also: api.PutApndArchArgs -------------------- terminology --------------------- here and elsewhere "archive" is any (.tar, .tgz/.tar.gz, .zip, .tar.lz4) formatted object. [NOTE] see cmn/api for cmn.ArchiveMsg (that also contains ToBck)
type BlobMsg ¶ added in v1.3.22
type BlobMsg struct { ChunkSize int64 `json:"chunk-size"` // as in: chunk size FullSize int64 `json:"full-size"` // user-specified (full) size of the object to download NumWorkers int `json:"num-workers"` // number of concurrent downloading workers (readers); `dfltNumWorkers` when zero LatestVer bool `json:"latest-ver"` // when true and in-cluster: check with remote whether (deleted | version-changed) }
type BsummCtrlMsg ¶ added in v1.3.18
type BsummCtrlMsg struct { UUID string `json:"uuid"` Prefix string `json:"prefix"` ObjCached bool `json:"cached"` BckPresent bool `json:"present"` DontAddRemote bool `json:"dont_add_remote"` }
to generate bucket summary (or summaries)
type BsummResult ¶ added in v1.3.18
type BsummResult struct { ObjCount struct { Present uint64 `json:"obj_count_present,string"` Remote uint64 `json:"obj_count_remote,string"` } ObjSize struct { Min int64 `json:"obj_min_size"` Avg int64 `json:"obj_avg_size"` Max int64 `json:"obj_max_size"` } TotalSize struct { OnDisk uint64 `json:"size_on_disk,string"` // sum(dir sizes) aka "apparent size" PresentObjs uint64 `json:"size_all_present_objs,string"` // sum(cached object sizes) RemoteObjs uint64 `json:"size_all_remote_objs,string"` // sum(all object sizes in a remote bucket) Disks uint64 `json:"total_disks_size,string"` } UsedPct uint64 `json:"used_pct"` IsBckPresent bool `json:"is_present"` // in BMD }
"summarized" result for a given bucket
type CapacityInfo ¶
type CapacityInfo struct { Used uint64 `json:"fs_used,string"` Total uint64 `json:"fs_capacity,string"` PctUsed float64 `json:"pct_fs_used"` }
sysinfo
type ClusterSysInfo ¶
type ClusterSysInfo struct { Proxy map[string]*MemCPUInfo `json:"proxy"` Target map[string]*TSysInfo `json:"target"` }
sysinfo
type ClusterSysInfoRaw ¶
type ClusterSysInfoRaw struct { Proxy cos.JSONRawMsgs `json:"proxy"` Target cos.JSONRawMsgs `json:"target"` }
sysinfo
type CopyBckMsg ¶
type CopyBckMsg struct { Prepend string `json:"prepend"` // destination naming, as in: dest-obj-name = Prepend + source-obj-name Prefix string `json:"prefix"` // prefix to select matching _source_ objects or virtual directories DryRun bool `json:"dry_run"` // visit all source objects, don't make any modifications Force bool `json:"force"` // force running in presence of "limited coexistence" type conflicts LatestVer bool `json:"latest-ver"` // see also: QparamLatestVer, 'versioning.validate_warm_get', PrefetchMsg Sync bool `json:"synchronize"` // see also: 'versioning.synchronize' }
copy & (offline) transform bucket to bucket
type JoinNodeResult ¶
type ListRange ¶ added in v1.3.18
List of object names _or_ a template specifying { optional Prefix, zero or more Ranges }
func (*ListRange) HasTemplate ¶ added in v1.3.18
type LsoMsg ¶
type LsoMsg struct { UUID string `json:"uuid"` // ID to identify a single multi-page request Props string `json:"props"` // comma-delimited, e.g. "checksum,size,custom" (see GetProps* enum) TimeFormat string `json:"time_format,omitempty"` // RFC822 is the default Prefix string `json:"prefix"` // return obj names starting with prefix (TODO: e.g. "A.tar/tutorials/") StartAfter string `json:"start_after,omitempty"` // start listing after (AIS buckets only) ContinuationToken string `json:"continuation_token"` // => LsoResult.ContinuationToken => LsoMsg.ContinuationToken SID string `json:"target"` // selected target to solely execute backend.list-objects Flags uint64 `json:"flags,string"` // enum {LsObjCached, ...} - "LsoMsg flags" above PageSize int64 `json:"pagesize"` // max entries returned by list objects call Header http.Header `json:"hdr,omitempty"` // (for pointers, see `ListArgs` in api/ls.go) }
func (*LsoMsg) WantOnlyRemoteProps ¶
type MemCPUInfo ¶ added in v1.3.16
type MemCPUInfo struct { MemUsed uint64 `json:"mem_used"` MemAvail uint64 `json:"mem_avail"` PctMemUsed float64 `json:"pct_mem_used"` PctCPUUsed float64 `json:"pct_cpu_used"` LoadAvg sys.LoadAvg `json:"load_avg"` }
func GetMemCPU ¶ added in v1.3.16
func GetMemCPU() MemCPUInfo
type MountpathList ¶
type MountpathList struct { Available []string `json:"available"` WaitingDD []string `json:"waiting_dd"` Disabled []string `json:"disabled"` }
MountpathList contains two lists:
- Available - list of local mountpaths available to the storage target
- WaitingDD - waiting for resilvering completion to be detached or disabled (moved to `Disabled`)
- Disabled - list of disabled mountpaths, the mountpaths that generated IO errors followed by (FSHC) health check, etc.
type PrefetchMsg ¶ added in v1.3.22
type PrefetchMsg struct { ListRange BlobThreshold int64 `json:"blob-threshold"` // when greater than threshold prefetch using blob-downloader; otherwise cold GET NumWorkers int `json:"num-workers"` // number of concurrent workers; 0 - number of mountpaths (default); (-1) none ContinueOnError bool `json:"coer"` // ignore non-critical errors, keep going LatestVer bool `json:"latest-ver"` // when true & in-cluster: check with remote whether (deleted | version-changed) }
prefetch
type PromoteArgs ¶ added in v1.3.22
type PromoteArgs struct { DaemonID string `json:"tid,omitempty"` // target ID SrcFQN string `json:"src,omitempty"` // source file or directory (must be absolute pathname) ObjName string `json:"obj,omitempty"` // destination object name or prefix Recursive bool `json:"rcr,omitempty"` // recursively promote nested dirs // once successfully promoted: OverwriteDst bool `json:"ovw,omitempty"` // overwrite destination DeleteSrc bool `json:"dls,omitempty"` // remove source when (and after) successfully promoting // and _not_ to try to auto-detect if it is; // (auto-detection takes time, etc.) SrcIsNotFshare bool `json:"notshr,omitempty"` // the source is not a file share equally accessible by all targets }
common part that's used in `api.PromoteArgs` and `PromoteParams`(server side), both
type TCBMsg ¶
type TCBMsg struct { // NOTE: objname extension ---------------------------------------------------------------------- // - resulting object names will have this extension, if specified. // - if source bucket has two (or more) objects with the same base name but different extension, // specifying this field might cause unintended override. // - this field might not be any longer required - TODO review Ext cos.StrKVs `json:"ext"` Transform CopyBckMsg }
copy & (offline) transform bucket to bucket
type TCOMsg ¶ added in v1.3.24
type TCOMsg struct { ListRange TxnUUID string // (plstcx client, internal use) NumWorkers int `json:"num-workers"` // number of concurrent workers; 0 - number of mountpaths (default); (-1) none TCBMsg ContinueOnError bool `json:"coer"` }
multi-object copy & transform [NOTE] see cmn/api for cmn.TCOMsg (that also contains ToBck); see also TCBMsg
type Transform ¶ added in v1.3.16
type Transform struct { Name string `json:"id,omitempty"` Timeout cos.Duration `json:"request_timeout,omitempty"` }
copy & (offline) transform bucket to bucket
type WritePolicy ¶
type WritePolicy string
write policy (enum and accessors) applies to both AIS metadata and data; bucket-configurable with global defaults via cluster config
func (WritePolicy) IsImmediate ¶
func (wp WritePolicy) IsImmediate() bool
func (WritePolicy) Validate ¶
func (wp WritePolicy) Validate() (err error)