downloader

package
v1.3.15 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 13, 2022 License: MIT Imports: 31 Imported by: 0

Documentation

Overview

Package cmn provides common low-level types and utilities for all aistore projects

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package cmn provides common low-level types and utilities for all aistore projects

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2021, NVIDIA CORPORATION. All rights reserved.

Package downloader implements functionality to download resources into AIS cluster from external source.

  • Copyright (c) 2018-2022, NVIDIA CORPORATION. All rights reserved.

Index

Constants

View Source 
const (
	DiffResolverSend = iota
	DiffResolverRecv
	DiffResolverDelete
	DiffResolverSkip
	DiffResolverErr
	DiffResolverEOF
)

Variables

This section is empty.

Functions

func CompareObjects 

func CompareObjects(lom *cluster.LOM, dst *DstElement) (equal bool, err error)

Use all available metadata including {size, version, ETag, MD5, CRC} to compare local object with its remote counterpart.

func IsType 

func IsType(a string) bool

func NormalizeObjName 

func NormalizeObjName(objName string) (string, error)

Removes everything that goes after '?', eg. "?query=key..." so it will not be part of final object name.

func SetDB 

func SetDB(dbdrv kvdb.Driver)

func Xreg 

func Xreg()

Types

type BackendResource 

type BackendResource struct {
	ObjName string
}

type DiffResolver 

type DiffResolver struct {
	// contains filtered or unexported fields
}

DiffResolver is entity that computes difference between two streams of objects. The streams are expected to be in sorted order.

func NewDiffResolver 

func NewDiffResolver(ctx DiffResolverCtx) *DiffResolver

func (*DiffResolver) Abort 

func (dr *DiffResolver) Abort(err error)

func (*DiffResolver) CloseDst 

func (dr *DiffResolver) CloseDst()

func (*DiffResolver) CloseSrc 

func (dr *DiffResolver) CloseSrc()

func (*DiffResolver) Next 

func (dr *DiffResolver) Next() (DiffResolverResult, error)

func (*DiffResolver) PushDst 

func (dr *DiffResolver) PushDst(v any)

func (*DiffResolver) PushSrc 

func (dr *DiffResolver) PushSrc(v any)

func (*DiffResolver) Start 

func (dr *DiffResolver) Start()

func (*DiffResolver) Stop 

func (dr *DiffResolver) Stop()

func (*DiffResolver) Stopped 

func (dr *DiffResolver) Stopped() bool

type DiffResolverCtx 

type DiffResolverCtx interface {
	CompareObjects(*cluster.LOM, *DstElement) (bool, error)
	IsObjFromRemote(*cluster.LOM) (bool, error)
}

type DiffResolverResult 

type DiffResolverResult struct {
	Action uint8
	Src    *cluster.LOM
	Dst    *DstElement
	Err    error
}

type DlAdminBody 

type DlAdminBody struct {
	ID              string `json:"id"`
	Regex           string `json:"regex"`
	OnlyActiveTasks bool   `json:"only_active_tasks"` // Skips detailed info about tasks finished/errored
}

Internal status/delete request body

func (*DlAdminBody) Validate 

func (b *DlAdminBody) Validate(requireID bool) error

type DlBackendBody 

type DlBackendBody struct {
	DlBase
	Sync   bool   `json:"sync"`
	Prefix string `json:"prefix"`
	Suffix string `json:"suffix"`
}

Backend download request

func (*DlBackendBody) Describe 

func (b *DlBackendBody) Describe() string

func (*DlBackendBody) Validate 

func (b *DlBackendBody) Validate() error

type DlBase 

type DlBase struct {
	Description      string   `json:"description"`
	Bck              cmn.Bck  `json:"bucket"`
	Timeout          string   `json:"timeout"`
	ProgressInterval string   `json:"progress_interval"`
	Limits           DlLimits `json:"limits"`
}

func (*DlBase) Validate 

func (b *DlBase) Validate() error

type DlBody 

type DlBody struct {
	Type DlType `json:"type"`
	json.RawMessage
}

NOTE: Changing this structure requires changes in `MarshalJSON` and `UnmarshalJSON` methods.

func (DlBody) MarshalJSON 

func (db DlBody) MarshalJSON() ([]byte, error)

func (*DlBody) UnmarshalJSON 

func (db *DlBody) UnmarshalJSON(b []byte) error

type DlJob 

type DlJob interface {
	ID() string
	Bck() *cmn.Bck
	Description() string
	Timeout() time.Duration
	ActiveStats() (*DlStatusResp, error)
	String() string
	Notif() cluster.Notif // notifications
	AddNotif(n cluster.Notif, job DlJob)

	// If total length (size) of download job is not known, -1 should be returned.
	Len() int

	// Determines if it requires also syncing.
	Sync() bool
	// contains filtered or unexported methods
}

func ParseStartDownloadRequest 

func ParseStartDownloadRequest(t cluster.Target, bck *cluster.Bck, id string, dlb DlBody, dlXact *Downloader) (DlJob, error)

type DlJobInfo 

type DlJobInfo struct {
	ID            string    `json:"id"`
	Description   string    `json:"description"`
	FinishedCnt   int       `json:"finished_cnt"`
	ScheduledCnt  int       `json:"scheduled_cnt"` // tasks being processed or already processed by dispatched
	SkippedCnt    int       `json:"skipped_cnt"`   // number of tasks skipped
	ErrorCnt      int       `json:"error_cnt"`
	Total         int       `json:"total"`          // total number of tasks, negative if unknown
	AllDispatched bool      `json:"all_dispatched"` // if true, dispatcher has already scheduled all tasks for given job
	Aborted       bool      `json:"aborted"`
	StartedTime   time.Time `json:"started_time"`
	FinishedTime  time.Time `json:"finished_time"`
}

Summary info of the download job

func (*DlJobInfo) Aggregate 

func (j *DlJobInfo) Aggregate(rhs *DlJobInfo)

func (DlJobInfo) DoneCnt 

func (j DlJobInfo) DoneCnt() int

DoneCnt returns number of tasks that have finished (either successfully or with an error).

func (DlJobInfo) JobFinished 

func (j DlJobInfo) JobFinished() bool

func (DlJobInfo) JobRunning 

func (j DlJobInfo) JobRunning() bool

func (DlJobInfo) PendingCnt 

func (j DlJobInfo) PendingCnt() int

PendingCnt returns number of tasks which are currently being processed.

func (DlJobInfo) String 

func (j DlJobInfo) String() string

func (DlJobInfo) TotalCnt 

func (j DlJobInfo) TotalCnt() int

type DlJobInfos 

type DlJobInfos []*DlJobInfo

func (DlJobInfos) Len 

func (d DlJobInfos) Len() int

func (DlJobInfos) Less 

func (d DlJobInfos) Less(i, j int) bool

func (DlJobInfos) Swap 

func (d DlJobInfos) Swap(i, j int)

type DlLimits 

type DlLimits struct {
	Connections  int `json:"connections"`
	BytesPerHour int `json:"bytes_per_hour"`
}

type DlMultiBody 

type DlMultiBody struct {
	DlBase
	ObjectsPayload any `json:"objects"`
}

Multi request

func (*DlMultiBody) Describe 

func (b *DlMultiBody) Describe() string

func (*DlMultiBody) ExtractPayload 

func (b *DlMultiBody) ExtractPayload() (cos.StrKVs, error)

func (*DlMultiBody) String 

func (b *DlMultiBody) String() string

func (*DlMultiBody) Validate 

func (b *DlMultiBody) Validate() error

type DlPostResp 

type DlPostResp struct {
	ID string `json:"id"`
}

Download POST result returned to the user

type DlRangeBody 

type DlRangeBody struct {
	DlBase
	Template string `json:"template"`
	Subdir   string `json:"subdir"`
}

Range request

func (*DlRangeBody) Describe 

func (b *DlRangeBody) Describe() string

func (*DlRangeBody) String 

func (b *DlRangeBody) String() string

func (*DlRangeBody) Validate 

func (b *DlRangeBody) Validate() error

type DlSingleBody 

type DlSingleBody struct {
	DlBase
	DlSingleObj
}

Single request

func (*DlSingleBody) Describe 

func (b *DlSingleBody) Describe() string

func (*DlSingleBody) ExtractPayload 

func (b *DlSingleBody) ExtractPayload() (cos.StrKVs, error)

func (*DlSingleBody) String 

func (b *DlSingleBody) String() string

func (*DlSingleBody) Validate 

func (b *DlSingleBody) Validate() error

type DlSingleObj 

type DlSingleObj struct {
	ObjName    string `json:"object_name"`
	Link       string `json:"link"`
	FromRemote bool   `json:"from_remote"`
}

func (*DlSingleObj) Validate 

func (b *DlSingleObj) Validate() error

type DlStatusResp 

type DlStatusResp struct {
	DlJobInfo
	CurrentTasks  []TaskDlInfo  `json:"current_tasks,omitempty"`
	FinishedTasks []TaskDlInfo  `json:"finished_tasks,omitempty"`
	Errs          []TaskErrInfo `json:"download_errors,omitempty"`
}

func (*DlStatusResp) Aggregate 

func (d *DlStatusResp) Aggregate(rhs DlStatusResp) *DlStatusResp

type DlType 

type DlType string
const (
	DlTypeSingle  DlType = "single"
	DlTypeRange   DlType = "range"
	DlTypeMulti   DlType = "multi"
	DlTypeBackend DlType = "backend"

	DownloadProgressInterval = 10 * time.Second
)

type Downloader 

type Downloader struct {
	xact.DemandBase
	// contains filtered or unexported fields
}

Downloader implements xact.Demand interface. Upon getting requests via AIS download endpoint, Downloader dispatches these requests to the corresponding jogger.

func (*Downloader) AbortJob 

func (d *Downloader) AbortJob(id string) (resp any, statusCode int, err error)

func (*Downloader) Download 

func (d *Downloader) Download(dJob DlJob) (resp any, statusCode int, err error)

func (*Downloader) JobStatus 

func (d *Downloader) JobStatus(id string, onlyActive bool) (resp any, statusCode int, err error)

func (*Downloader) ListJobs 

func (d *Downloader) ListJobs(regex *regexp.Regexp) (resp any, statusCode int, err error)

func (*Downloader) Name 

func (*Downloader) Name() string

func (*Downloader) RemoveJob 

func (d *Downloader) RemoveJob(id string) (resp any, statusCode int, err error)

func (*Downloader) Run 

func (d *Downloader) Run(*sync.WaitGroup)

func (*Downloader) Snap 

func (d *Downloader) Snap() cluster.XactSnap

type DstElement 

type DstElement struct {
	ObjName string
	Version string
	Link    string
}

type NotifDownload 

type NotifDownload struct {
	nl.NotifBase
	DlJob DlJob
}

func (*NotifDownload) ToNotifMsg 

func (nd *NotifDownload) ToNotifMsg() cluster.NotifMsg

type NotifDownloadListerner 

type NotifDownloadListerner struct {
	nl.NotifListenerBase
}

func NewDownloadNL 

func NewDownloadNL(uuid string, action string, smap *cluster.Smap,
	progressInterval time.Duration, bck ...*cmn.Bck) *NotifDownloadListerner

func (*NotifDownloadListerner) AbortArgs 

func (nd *NotifDownloadListerner) AbortArgs() cmn.HreqArgs

func (*NotifDownloadListerner) QueryArgs 

func (nd *NotifDownloadListerner) QueryArgs() cmn.HreqArgs

func (*NotifDownloadListerner) UnmarshalStats 

func (*NotifDownloadListerner) UnmarshalStats(rawMsg []byte) (stats any, finished, aborted bool, err error)

type TaskDlInfo 

type TaskDlInfo struct {
	Name       string    `json:"name"`
	Downloaded int64     `json:"downloaded,string"`
	Total      int64     `json:"total,string,omitempty"`
	StartTime  time.Time `json:"start_time,omitempty"`
	EndTime    time.Time `json:"end_time,omitempty"`
	Running    bool      `json:"running"`
}

Info about a task that is currently or has been downloaded by one of the joggers

type TaskErrByName 

type TaskErrByName []TaskErrInfo

func (TaskErrByName) Len 

func (t TaskErrByName) Len() int

func (TaskErrByName) Less 

func (t TaskErrByName) Less(i, j int) bool

func (TaskErrByName) Swap 

func (t TaskErrByName) Swap(i, j int)

type TaskErrInfo 

type TaskErrInfo struct {
	Name string `json:"name"`
	Err  string `json:"error"`
}

type TaskInfoByName 

type TaskInfoByName []TaskDlInfo

func (TaskInfoByName) Len 

func (t TaskInfoByName) Len() int

func (TaskInfoByName) Less 

func (t TaskInfoByName) Less(i, j int) bool

func (TaskInfoByName) Swap 

func (t TaskInfoByName) Swap(i, j int)

type WebResource 

type WebResource struct {
	ObjName string
	Link    string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.