Documentation ¶
Overview ¶
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package cmn provides common low-level types and utilities for all aistore projects
- Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Package dload implements functionality to download resources into AIS cluster from external source.
- Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.
Index ¶
- Constants
- func AbortReq(jobID string) cmn.HreqArgs
- func CompareObjects(lom *cluster.LOM, dst *DstElement) (equal bool, err error)
- func IsType(a string) bool
- func ListJobs(regex *regexp.Regexp, onlyActive bool) (any, int, error)
- func NormalizeObjName(objName string) (string, error)
- func ParseStartRequest(t cluster.Target, bck *cluster.Bck, id string, dlb Body, xdl *Xact) (jobif, error)
- func SetDB(dbdrv kvdb.Driver)
- func Xreg()
- type AdminBody
- type BackendBody
- type BackendResource
- type Base
- type Body
- type DiffResolver
- func (dr *DiffResolver) Abort(err error)
- func (dr *DiffResolver) CloseDst()
- func (dr *DiffResolver) CloseSrc()
- func (dr *DiffResolver) Next() (DiffResolverResult, error)
- func (dr *DiffResolver) PushDst(v any)
- func (dr *DiffResolver) PushSrc(v any)
- func (dr *DiffResolver) Start()
- func (dr *DiffResolver) Stop()
- func (dr *DiffResolver) Stopped() bool
- type DiffResolverCtx
- type DiffResolverResult
- type DlPostResp
- type DstElement
- type Job
- type JobInfos
- type Limits
- type MultiBody
- type NotifDownload
- type NotifDownloadListerner
- type RangeBody
- type SingleBody
- type SingleObj
- type StatusResp
- type TaskDlInfo
- type TaskErrByName
- type TaskErrInfo
- type TaskInfoByName
- type Type
- type WebResource
- type Xact
- func (xld *Xact) AbortJob(id string) (resp any, statusCode int, err error)
- func (xld *Xact) Download(job jobif) (resp any, statusCode int, err error)
- func (xld *Xact) JobStatus(id string, onlyActive bool) (resp any, statusCode int, err error)
- func (xld *Xact) RemoveJob(id string) (resp any, statusCode int, err error)
- func (xld *Xact) Run(*sync.WaitGroup)
- func (xld *Xact) Snap() (snap *cluster.Snap)
Constants ¶
const ( DiffResolverSend = iota DiffResolverRecv DiffResolverDelete DiffResolverSkip DiffResolverErr DiffResolverEOF )
const DownloadProgressInterval = 10 * time.Second
const PrefixJobID = "dnl-"
Variables ¶
This section is empty.
Functions ¶
func CompareObjects ¶
func CompareObjects(lom *cluster.LOM, dst *DstElement) (equal bool, err error)
Use all available metadata including {size, version, ETag, MD5, CRC} to compare local object with its remote counterpart.
func NormalizeObjName ¶
Removes everything that goes after '?', eg. "?query=key..." so it will not be part of final object name.
func ParseStartRequest ¶
Types ¶
type AdminBody ¶
type BackendBody ¶
type BackendBody struct { Base Prefix string `json:"prefix"` Suffix string `json:"suffix"` Sync bool `json:"sync"` }
func (*BackendBody) Describe ¶
func (b *BackendBody) Describe() string
func (*BackendBody) Validate ¶
func (b *BackendBody) Validate() error
type BackendResource ¶
type BackendResource struct {
ObjName string
}
type Base ¶
type Body ¶
type Body struct { Type Type `json:"type"` json.RawMessage }
NOTE: Changing this structure requires changes in `MarshalJSON` and `UnmarshalJSON` methods.
func (Body) MarshalJSON ¶
func (*Body) UnmarshalJSON ¶
type DiffResolver ¶
type DiffResolver struct {
// contains filtered or unexported fields
}
DiffResolver is entity that computes difference between two streams of objects. The streams are expected to be in sorted order.
func NewDiffResolver ¶
func NewDiffResolver(ctx DiffResolverCtx) *DiffResolver
func (*DiffResolver) Abort ¶
func (dr *DiffResolver) Abort(err error)
func (*DiffResolver) CloseDst ¶
func (dr *DiffResolver) CloseDst()
func (*DiffResolver) CloseSrc ¶
func (dr *DiffResolver) CloseSrc()
func (*DiffResolver) Next ¶
func (dr *DiffResolver) Next() (DiffResolverResult, error)
func (*DiffResolver) PushDst ¶
func (dr *DiffResolver) PushDst(v any)
func (*DiffResolver) PushSrc ¶
func (dr *DiffResolver) PushSrc(v any)
func (*DiffResolver) Start ¶
func (dr *DiffResolver) Start()
func (*DiffResolver) Stop ¶
func (dr *DiffResolver) Stop()
func (*DiffResolver) Stopped ¶
func (dr *DiffResolver) Stopped() bool
type DiffResolverCtx ¶
type DiffResolverResult ¶
type DiffResolverResult struct { Action uint8 Src *cluster.LOM Dst *DstElement Err error }
type DlPostResp ¶
type DlPostResp struct {
ID string `json:"id"`
}
Download POST result returned to the user
type DstElement ¶
type Job ¶
type Job struct { ID string `json:"id"` XactID string `json:"xaction_id"` Description string `json:"description"` StartedTime time.Time `json:"started_time"` FinishedTime time.Time `json:"finished_time"` FinishedCnt int `json:"finished_cnt"` ScheduledCnt int `json:"scheduled_cnt"` // tasks being processed or already processed by dispatched SkippedCnt int `json:"skipped_cnt"` // number of tasks skipped ErrorCnt int `json:"error_cnt"` Total int `json:"total"` // total number of tasks, negative if unknown AllDispatched bool `json:"all_dispatched"` // if true, dispatcher has already scheduled all tasks for given job Aborted bool `json:"aborted"` }
func (*Job) DoneCnt ¶
DoneCnt returns number of tasks that have finished (either successfully or with an error).
func (*Job) JobFinished ¶
func (*Job) JobRunning ¶
func (*Job) PendingCnt ¶
PendingCnt returns number of tasks which are currently being processed.
type NotifDownload ¶
func (*NotifDownload) ToNotifMsg ¶
func (nd *NotifDownload) ToNotifMsg() cluster.NotifMsg
type NotifDownloadListerner ¶
type NotifDownloadListerner struct {
nl.ListenerBase
}
func NewDownloadNL ¶
func (*NotifDownloadListerner) QueryArgs ¶
func (nd *NotifDownloadListerner) QueryArgs() cmn.HreqArgs
func (*NotifDownloadListerner) UnmarshalStats ¶
func (*NotifDownloadListerner) UnmarshalStats(rawMsg []byte) (stats any, finished, aborted bool, err error)
type SingleBody ¶
func (*SingleBody) Describe ¶
func (b *SingleBody) Describe() string
func (*SingleBody) ExtractPayload ¶
func (b *SingleBody) ExtractPayload() (cos.StrKVs, error)
func (*SingleBody) String ¶
func (b *SingleBody) String() string
func (*SingleBody) Validate ¶
func (b *SingleBody) Validate() error
type SingleObj ¶
type StatusResp ¶
type StatusResp struct { Job CurrentTasks []TaskDlInfo `json:"current_tasks,omitempty"` FinishedTasks []TaskDlInfo `json:"finished_tasks,omitempty"` Errs []TaskErrInfo `json:"download_errors,omitempty"` }
func (*StatusResp) Aggregate ¶
func (d *StatusResp) Aggregate(rhs StatusResp) *StatusResp
type TaskDlInfo ¶
type TaskErrByName ¶
type TaskErrByName []TaskErrInfo
func (TaskErrByName) Len ¶
func (t TaskErrByName) Len() int
func (TaskErrByName) Less ¶
func (t TaskErrByName) Less(i, j int) bool
func (TaskErrByName) Swap ¶
func (t TaskErrByName) Swap(i, j int)
type TaskErrInfo ¶
type TaskInfoByName ¶
type TaskInfoByName []TaskDlInfo
func (TaskInfoByName) Len ¶
func (t TaskInfoByName) Len() int
func (TaskInfoByName) Less ¶
func (t TaskInfoByName) Less(i, j int) bool
func (TaskInfoByName) Swap ¶
func (t TaskInfoByName) Swap(i, j int)