sources

Documentation

Index

• func NewAzureBlob(params *AzureBlobParams) []pipeline.SourcePipeline
• func NewHTTP(sourceURIs []string, targetAliases []string, md5 bool) pipeline.SourcePipeline
• func NewMultiFile(params *MultiFileParams) []pipeline.SourcePipeline
• type AzureBlob
  • func (f *AzureBlob) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *AzureBlob) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *AzureBlob) GetSourcesInfo() []pipeline.SourceInfo
• type AzureBlobParams
• type FileInfo
• type HTTPPipeline
  • func (f *HTTPPipeline) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *HTTPPipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *HTTPPipeline) GetSourcesInfo() []pipeline.SourceInfo
• type MultiFileParams
• type MultiFilePipeline
  • func (f *MultiFilePipeline) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *MultiFilePipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, ...)
  • func (f *MultiFilePipeline) GetSourcesInfo() []pipeline.SourceInfo

Functions

func NewAzureBlob

func NewAzureBlob(params *AzureBlobParams) []pipeline.SourcePipeline

NewAzureBlob creates a new instance of HTTPPipeline for Azure Blobs

func NewHTTP

func NewHTTP(sourceURIs []string, targetAliases []string, md5 bool) pipeline.SourcePipeline

NewHTTP creates a new instance of an HTTP source To get the file size, a HTTP HEAD request is issued and the Content-Length header is inspected.

func NewMultiFile

func NewMultiFile(params *MultiFileParams) []pipeline.SourcePipeline

NewMultiFile creates a new MultiFilePipeline. If the sourcePattern results in a single file, the targetAlias, if set, will be used as the target name. Otherwise the full original file name will be used instead. sourcePatterns []string, blockSize uint64, targetAliases []string, numOfPartitions int, md5 bool

Types

type AzureBlob

type AzureBlob struct {
	HTTPSource HTTPPipeline
	Container  string
	BlobNames  []string
	// contains filtered or unexported fields
}

AzureBlob constructs parts channel and implements data readers for Azure Blobs exposed via HTTP

func (*AzureBlob) ConstructBlockInfoQueue

func (f *AzureBlob) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, numOfBlocks int, size uint64)

ConstructBlockInfoQueue implements GetSourcesInfo from the pipeline.SourcePipeline Interface. Constructs the Part's channel arithmetically from the size of the sources.

func (*AzureBlob) ExecuteReader

func (f *AzureBlob) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, readPartsQ chan pipeline.Part, id int, wg *sync.WaitGroup)

ExecuteReader implements ExecuteReader from the pipeline.SourcePipeline Interface. For each part the reader makes a byte range request to the source starting from the part's Offset to BytesToRead - 1 (zero based).

func (*AzureBlob) GetSourcesInfo

func (f *AzureBlob) GetSourcesInfo() []pipeline.SourceInfo

GetSourcesInfo implements GetSourcesInfo from the pipeline.SourcePipeline Interface.

type AzureBlobParams

type AzureBlobParams struct {
	Container         string
	BlobNames         []string
	AccountName       string
	AccountKey        string
	CalculateMD5      bool
	UseExactNameMatch bool
	KeepDirStructure  bool
	FilesPerPipeline  int
}

AzureBlobParams TODO

type FileInfo

type FileInfo struct {
	FileStats   *os.FileInfo
	SourceURI   string
	TargetAlias string
	NumOfBlocks int
}

FileInfo Contains the metadata associated with a file to be transferred

type HTTPPipeline

type HTTPPipeline struct {
	Sources    []pipeline.SourceInfo
	HTTPClient *http.Client
	// contains filtered or unexported fields
}

HTTPPipeline constructs parts channel and implements data readers for file exposed via HTTP

func (*HTTPPipeline) ConstructBlockInfoQueue

func (f *HTTPPipeline) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, numOfBlocks int, size uint64)

ConstructBlockInfoQueue implements GetSourcesInfo from the pipeline.SourcePipeline Interface. Constructs the Part's channel arithmetically from the size of the sources.

func (*HTTPPipeline) ExecuteReader

func (f *HTTPPipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, readPartsQ chan pipeline.Part, id int, wg *sync.WaitGroup)

ExecuteReader implements ExecuteReader from the pipeline.SourcePipeline Interface. For each part the reader makes a byte range request to the source starting from the part's Offset to BytesToRead - 1 (zero based).

func (*HTTPPipeline) GetSourcesInfo

func (f *HTTPPipeline) GetSourcesInfo() []pipeline.SourceInfo

GetSourcesInfo implements GetSourcesInfo from the pipeline.SourcePipeline Interface. Returns an array of pipeline.SourceInfo[] with the files URL, alias and size.

type MultiFileParams

type MultiFileParams struct {
	SourcePatterns   []string
	BlockSize        uint64
	TargetAliases    []string
	NumOfPartitions  int
	MD5              bool
	FilesPerPipeline int
	KeepDirStructure bool
}

MultiFileParams TODO

type MultiFilePipeline

type MultiFilePipeline struct {
	FilesInfo           map[string]FileInfo
	TotalNumberOfBlocks int
	TotalSize           uint64
	BlockSize           uint64
	NumOfPartitions     int
	// contains filtered or unexported fields
}

MultiFilePipeline Contructs blocks queue and implements data readers

func (*MultiFilePipeline) ConstructBlockInfoQueue

func (f *MultiFilePipeline) ConstructBlockInfoQueue(blockSize uint64) (partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, numOfBlocks int, size uint64)

ConstructBlockInfoQueue implements ConstructBlockInfoQueue from the pipeline.SourcePipeline Interface. this implementation uses partitions to group parts into a set that can be read sequentially. This is to avoid Window's memory pressure when calling SetFilePointer numerous times on the same handle

func (*MultiFilePipeline) ExecuteReader

func (f *MultiFilePipeline) ExecuteReader(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, readPartsQ chan pipeline.Part, id int, wg *sync.WaitGroup)

ExecuteReader implements ExecuteReader from the pipeline.SourcePipeline Interface. For each file the reader will maintain a open handle from which data will be read. This implementation uses partitions (group of parts that can be read sequentially).

func (*MultiFilePipeline) GetSourcesInfo

func (f *MultiFilePipeline) GetSourcesInfo() []pipeline.SourceInfo

GetSourcesInfo implements GetSourcesInfo from the pipeline.SourcePipeline Interface. Returns an an array of SourceInfo with the name, alias and size of the files to be transferred.