dload

package
v1.3.16 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 31, 2023 License: MIT Imports: 30 Imported by: 1

Documentation

Overview

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package cmn provides common low-level types and utilities for all aistore projects

  • Copyright (c) 2018-2023, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package dload implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source 
const (
	DiffResolverSend = iota
	DiffResolverRecv
	DiffResolverDelete
	DiffResolverSkip
	DiffResolverErr
	DiffResolverEOF
)
View Source 
const DownloadProgressInterval = 10 * time.Second
View Source 
const PrefixJobID = "dnl-"

Variables

This section is empty.

Functions

func AbortReq 

func AbortReq(jobID string) cmn.HreqArgs

called via ais/prxnotifs generic mechanism

func CompareObjects 

func CompareObjects(lom *cluster.LOM, dst *DstElement) (equal bool, err error)

Use all available metadata including {size, version, ETag, MD5, CRC} to compare local object with its remote counterpart.

func IsType 

func IsType(a string) bool

func ListJobs 

func ListJobs(regex *regexp.Regexp, onlyActive bool) (any, int, error)

func NormalizeObjName 

func NormalizeObjName(objName string) (string, error)

Removes everything that goes after '?', eg. "?query=key..." so it will not be part of final object name.

func ParseStartRequest 

func ParseStartRequest(t cluster.Target, bck *cluster.Bck, id string, dlb Body, xdl *Xact) (jobif, error)

func SetDB 

func SetDB(dbdrv kvdb.Driver)

func Xreg 

func Xreg()

Types

type AdminBody 

type AdminBody struct {
	ID         string `json:"id"`
	Regex      string `json:"regex"`
	OnlyActive bool   `json:"only_active_tasks"` // Skips detailed info about tasks finished/errored
}

func (*AdminBody) Validate 

func (b *AdminBody) Validate(requireID bool) error

type BackendBody 

type BackendBody struct {
	Base
	Prefix string `json:"prefix"`
	Suffix string `json:"suffix"`
	Sync   bool   `json:"sync"`
}

func (*BackendBody) Describe 

func (b *BackendBody) Describe() string

func (*BackendBody) Validate 

func (b *BackendBody) Validate() error

type BackendResource 

type BackendResource struct {
	ObjName string
}

type Base 

type Base struct {
	Description      string  `json:"description"`
	Bck              cmn.Bck `json:"bucket"`
	Timeout          string  `json:"timeout"`
	ProgressInterval string  `json:"progress_interval"`
	Limits           Limits  `json:"limits"`
}

func (*Base) Validate 

func (b *Base) Validate() error

type Body 

type Body struct {
	Type Type `json:"type"`
	json.RawMessage
}

NOTE: Changing this structure requires changes in `MarshalJSON` and `UnmarshalJSON` methods.

func (Body) MarshalJSON 

func (db Body) MarshalJSON() ([]byte, error)

func (*Body) UnmarshalJSON 

func (db *Body) UnmarshalJSON(b []byte) error

type DiffResolver 

type DiffResolver struct {
	// contains filtered or unexported fields
}

DiffResolver is entity that computes difference between two streams of objects. The streams are expected to be in sorted order.

func NewDiffResolver 

func NewDiffResolver(ctx DiffResolverCtx) *DiffResolver

func (*DiffResolver) Abort 

func (dr *DiffResolver) Abort(err error)

func (*DiffResolver) CloseDst 

func (dr *DiffResolver) CloseDst()

func (*DiffResolver) CloseSrc 

func (dr *DiffResolver) CloseSrc()

func (*DiffResolver) Next 

func (dr *DiffResolver) Next() (DiffResolverResult, error)

func (*DiffResolver) PushDst 

func (dr *DiffResolver) PushDst(v any)

func (*DiffResolver) PushSrc 

func (dr *DiffResolver) PushSrc(v any)

func (*DiffResolver) Start 

func (dr *DiffResolver) Start()

func (*DiffResolver) Stop 

func (dr *DiffResolver) Stop()

func (*DiffResolver) Stopped 

func (dr *DiffResolver) Stopped() bool

type DiffResolverCtx 

type DiffResolverCtx interface {
	CompareObjects(*cluster.LOM, *DstElement) (bool, error)
	IsObjFromRemote(*cluster.LOM) (bool, error)
}

type DiffResolverResult 

type DiffResolverResult struct {
	Action uint8
	Src    *cluster.LOM
	Dst    *DstElement
	Err    error
}

type DlPostResp 

type DlPostResp struct {
	ID string `json:"id"`
}

Download POST result returned to the user

type DstElement 

type DstElement struct {
	ObjName string
	Version string
	Link    string
}

type Job 

type Job struct {
	ID            string    `json:"id"`
	XactID        string    `json:"xaction_id"`
	Description   string    `json:"description"`
	StartedTime   time.Time `json:"started_time"`
	FinishedTime  time.Time `json:"finished_time"`
	FinishedCnt   int       `json:"finished_cnt"`
	ScheduledCnt  int       `json:"scheduled_cnt"` // tasks being processed or already processed by dispatched
	SkippedCnt    int       `json:"skipped_cnt"`   // number of tasks skipped
	ErrorCnt      int       `json:"error_cnt"`
	Total         int       `json:"total"`          // total number of tasks, negative if unknown
	AllDispatched bool      `json:"all_dispatched"` // if true, dispatcher has already scheduled all tasks for given job
	Aborted       bool      `json:"aborted"`
}

func (*Job) Aggregate 

func (j *Job) Aggregate(rhs *Job)

func (*Job) DoneCnt 

func (j *Job) DoneCnt() int

DoneCnt returns number of tasks that have finished (either successfully or with an error).

func (*Job) JobFinished 

func (j *Job) JobFinished() bool

func (*Job) JobRunning 

func (j *Job) JobRunning() bool

func (*Job) PendingCnt 

func (j *Job) PendingCnt() int

PendingCnt returns number of tasks which are currently being processed.

func (*Job) String 

func (j *Job) String() string

func (*Job) TotalCnt 

func (j *Job) TotalCnt() int

type JobInfos 

type JobInfos []*Job

func (JobInfos) Len 

func (d JobInfos) Len() int

func (JobInfos) Less 

func (d JobInfos) Less(i, j int) bool

func (JobInfos) Swap 

func (d JobInfos) Swap(i, j int)

type Limits 

type Limits struct {
	Connections  int `json:"connections"`
	BytesPerHour int `json:"bytes_per_hour"`
}

type MultiBody 

type MultiBody struct {
	Base
	ObjectsPayload any `json:"objects"`
}

func (*MultiBody) Describe 

func (b *MultiBody) Describe() string

func (*MultiBody) ExtractPayload 

func (b *MultiBody) ExtractPayload() (cos.StrKVs, error)

func (*MultiBody) String 

func (b *MultiBody) String() string

func (*MultiBody) Validate 

func (b *MultiBody) Validate() error

type NotifDownload 

type NotifDownload struct {
	nl.Base
	// contains filtered or unexported fields
}

func (*NotifDownload) ToNotifMsg 

func (nd *NotifDownload) ToNotifMsg() cluster.NotifMsg

type NotifDownloadListerner 

type NotifDownloadListerner struct {
	nl.ListenerBase
}

func NewDownloadNL 

func NewDownloadNL(jobID, action string, smap *cluster.Smap, progressInterval time.Duration) *NotifDownloadListerner

func (*NotifDownloadListerner) QueryArgs 

func (nd *NotifDownloadListerner) QueryArgs() cmn.HreqArgs

func (*NotifDownloadListerner) UnmarshalStats 

func (*NotifDownloadListerner) UnmarshalStats(rawMsg []byte) (stats any, finished, aborted bool, err error)

type RangeBody 

type RangeBody struct {
	Base
	Template string `json:"template"`
	Subdir   string `json:"subdir"`
}

func (*RangeBody) Describe 

func (b *RangeBody) Describe() string

func (*RangeBody) String 

func (b *RangeBody) String() string

func (*RangeBody) Validate 

func (b *RangeBody) Validate() error

type SingleBody 

type SingleBody struct {
	Base
	SingleObj
}

func (*SingleBody) Describe 

func (b *SingleBody) Describe() string

func (*SingleBody) ExtractPayload 

func (b *SingleBody) ExtractPayload() (cos.StrKVs, error)

func (*SingleBody) String 

func (b *SingleBody) String() string

func (*SingleBody) Validate 

func (b *SingleBody) Validate() error

type SingleObj 

type SingleObj struct {
	ObjName    string `json:"object_name"`
	Link       string `json:"link"`
	FromRemote bool   `json:"from_remote"`
}

func (*SingleObj) Validate 

func (b *SingleObj) Validate() error

type StatusResp 

type StatusResp struct {
	Job
	CurrentTasks  []TaskDlInfo  `json:"current_tasks,omitempty"`
	FinishedTasks []TaskDlInfo  `json:"finished_tasks,omitempty"`
	Errs          []TaskErrInfo `json:"download_errors,omitempty"`
}

func (*StatusResp) Aggregate 

func (d *StatusResp) Aggregate(rhs StatusResp) *StatusResp

type TaskDlInfo 

type TaskDlInfo struct {
	Name       string    `json:"name"`
	Downloaded int64     `json:"downloaded,string"`
	Total      int64     `json:"total,string,omitempty"`
	StartTime  time.Time `json:"start_time,omitempty"`
	EndTime    time.Time `json:"end_time,omitempty"`
	Running    bool      `json:"running"`
}

type TaskErrByName 

type TaskErrByName []TaskErrInfo

func (TaskErrByName) Len 

func (t TaskErrByName) Len() int

func (TaskErrByName) Less 

func (t TaskErrByName) Less(i, j int) bool

func (TaskErrByName) Swap 

func (t TaskErrByName) Swap(i, j int)

type TaskErrInfo 

type TaskErrInfo struct {
	Name string `json:"name"`
	Err  string `json:"error"`
}

type TaskInfoByName 

type TaskInfoByName []TaskDlInfo

func (TaskInfoByName) Len 

func (t TaskInfoByName) Len() int

func (TaskInfoByName) Less 

func (t TaskInfoByName) Less(i, j int) bool

func (TaskInfoByName) Swap 

func (t TaskInfoByName) Swap(i, j int)

type Type 

type Type string
const (
	TypeSingle  Type = "single"
	TypeRange   Type = "range"
	TypeMulti   Type = "multi"
	TypeBackend Type = "backend"
)

type WebResource 

type WebResource struct {
	ObjName string
	Link    string
}

type Xact 

type Xact struct {
	xact.DemandBase
	// contains filtered or unexported fields
}

func (*Xact) AbortJob 

func (xld *Xact) AbortJob(id string) (resp any, statusCode int, err error)

func (*Xact) Download 

func (xld *Xact) Download(job jobif) (resp any, statusCode int, err error)

func (*Xact) JobStatus 

func (xld *Xact) JobStatus(id string, onlyActive bool) (resp any, statusCode int, err error)

func (*Xact) RemoveJob 

func (xld *Xact) RemoveJob(id string) (resp any, statusCode int, err error)

func (*Xact) Run 

func (xld *Xact) Run(*sync.WaitGroup)

func (*Xact) Snap 

func (xld *Xact) Snap() (snap *cluster.Snap)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.