transfer

Documentation

Index

• Constants
• func ParseTransferSegment(def Definition) (TransferSegment, TransferSegment)
• type Channels
• type Definition
  • func ParseTransferDefinition(str string) (Definition, error)
• type DupeCheckLevel
  • func ParseDupeCheckLevel(str string) (res DupeCheckLevel, err error)
  • func (d *DupeCheckLevel) ToString() string
• type ProgressUpdate
• type StatInfo
• type Stats
  • func NewStats(numberOfWorkers int, numberOfReaders int) *Stats
  • func (t *Stats) AddTransferInfo(info *StatInfo)
  • func (t *Stats) DisplaySummary()
• type TimeStatsInfo
• type Transfer
  • func NewTransfer(source pipeline.SourcePipeline, target pipeline.TargetPipeline, readers int, ...) *Transfer
  • func (t *Transfer) GetStats() *StatInfo
  • func (t *Transfer) SetTransferTracker(tracker *internal.TransferTracker)
  • func (t *Transfer) StartTransfer(dupeLevel DupeCheckLevel, progressBarDelegate ProgressUpdate)
  • func (t *Transfer) WaitForCompletion() (time.Duration, time.Duration)
• type TransferSegment
• type WaitGroups
• type Worker

Constants

const (
	FileToBlock Definition = "file-blockblob"
	HTTPToBlock            = "http-blockblob"
	BlobToFile             = "blob-file"
	FileToPage             = "file-pageblob"
	HTTPToPage             = "http-pageblob"
	HTTPToFile             = "http-file"
	BlobToBlock            = "blob-blockblob"
	BlobToPage             = "blob-pageblob"
	S3ToBlock              = "s3-blockblob"
	S3ToPage               = "s3-pageblob"
	PerfToBlock            = "perf-blockblob"
	PerfToPage             = "perf-pageblob"
	BlobToPerf             = "blob-perf"
)

The different transfers types

const (
	File      TransferSegment = "file"
	HTTP                      = "http"
	BlockBlob                 = "blockblob"
	PageBlob                  = "pageblob"
	S3                        = "s3"
	Perf                      = "perf"
	Blob                      = "blob"
	NA                        = "na"
)

valid transfer segments

const DupeCheckLevelStr = "None, ZeroOnly, Full" //TODO: package this better so it can stay in sync w/declaration

DupeCheckLevelStr list of duplicate blob check strategies

Functions

func ParseTransferSegment

func ParseTransferSegment(def Definition) (TransferSegment, TransferSegment)

ParseTransferSegment TODO

Types

type Channels

type Channels struct {
	ReadParts  chan pipeline.Part
	Parts      chan pipeline.Part
	Results    chan pipeline.WorkerResult
	Partitions chan pipeline.PartsPartition
}

Channels represents all the control channels in the transfer.

type Definition

type Definition string

Definition represents the supported source and target combinations

func ParseTransferDefinition

func ParseTransferDefinition(str string) (Definition, error)

ParseTransferDefinition parses a Definition from a string.

type DupeCheckLevel

type DupeCheckLevel int

DupeCheckLevel -- degree to which we'll try to check for duplicate blocks

const (
	None     DupeCheckLevel = iota // Don't bother to check
	ZeroOnly                       // Only check for blocks with all zero bytes
	Full                           // Compute MD5s and look for matches on that
)

The different levels for duplicate blocks

func ParseDupeCheckLevel

func ParseDupeCheckLevel(str string) (res DupeCheckLevel, err error)

ParseDupeCheckLevel converts string to DupeCheckLevel

func (*DupeCheckLevel) ToString

func (d *DupeCheckLevel) ToString() string

ToString printable representation of duplicate level values. For now, handle error as fatal, shouldn't be possible

type ProgressUpdate

type ProgressUpdate func(results pipeline.WorkerResult, committedCount int, bufferSize int)

ProgressUpdate called whenever a worker completes successfully

type StatInfo

type StatInfo struct {
	NumberOfFiles       int
	Duration            time.Duration
	TargetRetries       int32
	TotalNumberOfBlocks int
	TotalSize           uint64
	CumWriteDuration    time.Duration
}

StatInfo holds calculated statistics from a transfer

type Stats

type Stats struct {
	// contains filtered or unexported fields
}

Stats statistics from the transfer

func NewStats

func NewStats(numberOfWorkers int, numberOfReaders int) *Stats

NewStats creates a new instance of the

func (*Stats) AddTransferInfo

func (t *Stats) AddTransferInfo(info *StatInfo)

AddTransferInfo adds final stastistics info the list

func (*Stats) DisplaySummary

func (t *Stats) DisplaySummary()

DisplaySummary displays the final statistics for the transfer

type TimeStatsInfo

type TimeStatsInfo struct {
	StartTime        time.Time
	Duration         time.Duration
	CumWriteDuration time.Duration
}

TimeStatsInfo contains transfer statistics, used at the end of the transfer

type Transfer

type Transfer struct {
	SourcePipeline   pipeline.SourcePipeline
	TargetPipeline   pipeline.TargetPipeline
	NumOfReaders     int
	NumOfWorkers     int
	TotalNumOfBlocks int
	TotalSize        uint64
	ThreadTarget     int
	SyncWaitGroups   *WaitGroups
	ControlChannels  *Channels
	TimeStats        *TimeStatsInfo
	// contains filtered or unexported fields
}

Transfer top data structure holding the state of the transfer.

func NewTransfer

func NewTransfer(source pipeline.SourcePipeline, target pipeline.TargetPipeline, readers int, workers int, blockSize uint64) *Transfer

NewTransfer creates a new Transfer, this will adjust the thread target and initialize the channels and the wait groups for the writers, readers and the committers

func (*Transfer) GetStats

func (t *Transfer) GetStats() *StatInfo

GetStats returns the statistics of the transfer.

func (*Transfer) SetTransferTracker

func (t *Transfer) SetTransferTracker(tracker *internal.TransferTracker)

SetTransferTracker TODO

func (*Transfer) StartTransfer

func (t *Transfer) StartTransfer(dupeLevel DupeCheckLevel, progressBarDelegate ProgressUpdate)

StartTransfer starts the readers and readers. Process and commits the results of the write operations. The duration timer is started.

func (*Transfer) WaitForCompletion

func (t *Transfer) WaitForCompletion() (time.Duration, time.Duration)

WaitForCompletion blocks until the readers, writers and commit operations complete. Duration property is set and returned.

type TransferSegment

type TransferSegment string

TransferSegment source and target types

type WaitGroups

type WaitGroups struct {
	Readers sync.WaitGroup
	Workers sync.WaitGroup
	Commits sync.WaitGroup
}

WaitGroups holds all wait groups involved in the transfer.

type Worker

type Worker struct {
	WorkerID    int
	WorkerQueue chan pipeline.Part
	Result      chan pipeline.WorkerResult
	Wg          *sync.WaitGroup
	// contains filtered or unexported fields
}

Worker represents a worker routine that transfers data to a target