scraper

Documentation

Overview

Package scraper is a generated GoMock package.

Package scraper is a generated GoMock package.

Package scraper is a generated GoMock package.

Index

• type ArchiveFilesystemExtractor
  • func NewArchiveFilesystemExtractor(fs filesystem.FileSystem, opts ...ArchiveFilesystemExtractorOpts) *ArchiveFilesystemExtractor
  • func (e *ArchiveFilesystemExtractor) Extract(ctx context.Context, paths []string, process ProcessFn) error
• type ArchiveFilesystemExtractorOpts
  • func GenerateTarballMetaFile() ArchiveFilesystemExtractorOpts
• type DataType
• type ExtractLoadScraper
  • func NewExtractLoadScraper(extractor Extractor, loader Uploader) *ExtractLoadScraper
  • func (s *ExtractLoadScraper) Close() error
  • func (s *ExtractLoadScraper) Scrape(ctx context.Context, paths []string, execution testkube.Execution) error
• type Extractor
• type FileStat
• type FilesMeta
• type MinIOUploader
  • func NewMinIOUploader(endpoint, accessKeyID, secretAccessKey, region, token, bucket string, ssl bool) (*MinIOUploader, error)
  • func (l *MinIOUploader) Close() error
  • func (l *MinIOUploader) Upload(ctx context.Context, object *Object, execution testkube.Execution) error
• type MockExtractor
  • func NewMockExtractor(ctrl *gomock.Controller) *MockExtractor
  • func (m *MockExtractor) EXPECT() *MockExtractorMockRecorder
  • func (m *MockExtractor) Extract(arg0 context.Context, arg1 ProcessFn) error
• type MockExtractorMockRecorder
  • func (mr *MockExtractorMockRecorder) Extract(arg0, arg1 interface{}) *gomock.Call
• type MockScraper
  • func NewMockScraper(ctrl *gomock.Controller) *MockScraper
  • func (m *MockScraper) Close() error
  • func (m *MockScraper) EXPECT() *MockScraperMockRecorder
  • func (m *MockScraper) Scrape(arg0 context.Context, arg1 []string, arg2 testkube.Execution) error
• type MockScraperMockRecorder
  • func (mr *MockScraperMockRecorder) Close() *gomock.Call
  • func (mr *MockScraperMockRecorder) Scrape(arg0, arg1, arg2 interface{}) *gomock.Call
• type MockUploader
  • func NewMockUploader(ctrl *gomock.Controller) *MockUploader
  • func (m *MockUploader) Close() error
  • func (m *MockUploader) EXPECT() *MockUploaderMockRecorder
  • func (m *MockUploader) Upload(arg0 context.Context, arg1 *Object, arg2 testkube.Execution) error
• type MockUploaderMockRecorder
  • func (mr *MockUploaderMockRecorder) Close() *gomock.Call
  • func (mr *MockUploaderMockRecorder) Upload(arg0, arg1, arg2 interface{}) *gomock.Call
• type Object
• type ProcessFn
• type RecursiveFilesystemExtractor
  • func NewRecursiveFilesystemExtractor(fs filesystem.FileSystem) *RecursiveFilesystemExtractor
  • func (e *RecursiveFilesystemExtractor) Extract(ctx context.Context, paths []string, process ProcessFn) error
• type Scraper
• type Uploader

Types

type ArchiveFilesystemExtractor

type ArchiveFilesystemExtractor struct {
	// contains filtered or unexported fields
}

func NewArchiveFilesystemExtractor

func NewArchiveFilesystemExtractor(fs filesystem.FileSystem, opts ...ArchiveFilesystemExtractorOpts) *ArchiveFilesystemExtractor

func (*ArchiveFilesystemExtractor) Extract

func (e *ArchiveFilesystemExtractor) Extract(ctx context.Context, paths []string, process ProcessFn) error

type ArchiveFilesystemExtractorOpts

type ArchiveFilesystemExtractorOpts func(*ArchiveFilesystemExtractor)

func GenerateTarballMetaFile

func GenerateTarballMetaFile() ArchiveFilesystemExtractorOpts

type DataType

type DataType string

const (
	// DataTypeRaw specifies that the object is a raw file
	DataTypeRaw DataType = "raw"
	// DataTypeTarball specifies that the object is a tarball (gzip compressed tar archive)
	DataTypeTarball DataType = "tarball"
)

type ExtractLoadScraper

type ExtractLoadScraper struct {
	// contains filtered or unexported fields
}

func NewExtractLoadScraper

func NewExtractLoadScraper(extractor Extractor, loader Uploader) *ExtractLoadScraper

func (*ExtractLoadScraper) Close

func (s *ExtractLoadScraper) Close() error

func (*ExtractLoadScraper) Scrape

func (s *ExtractLoadScraper) Scrape(ctx context.Context, paths []string, execution testkube.Execution) error

type Extractor

type Extractor interface {
	Extract(ctx context.Context, paths []string, process ProcessFn) error
}

type FileStat

type FileStat struct {
	Name string `json:"name"`
	Size int64  `json:"size"`
}

type FilesMeta

type FilesMeta struct {
	// DataType is the type of data that is stored
	DataType DataType `json:"dataType"`
	// Files is a list of files that are stored and their original sizes
	Files []*FileStat `json:"files"`
	// Archive is the name of the archive file that contains all the files
	Archive string `json:"archive,omitempty"`
}

type MinIOUploader

type MinIOUploader struct {
	Endpoint, AccessKeyID, SecretAccessKey, Region, Token, Bucket string
	Ssl                                                           bool
	// contains filtered or unexported fields
}

func NewMinIOUploader

func NewMinIOUploader(endpoint, accessKeyID, secretAccessKey, region, token, bucket string, ssl bool) (*MinIOUploader, error)

func (*MinIOUploader) Close

func (l *MinIOUploader) Close() error

func (*MinIOUploader) Upload

func (l *MinIOUploader) Upload(ctx context.Context, object *Object, execution testkube.Execution) error

type MockExtractor

type MockExtractor struct {
	// contains filtered or unexported fields
}

MockExtractor is a mock of Extractor interface.

func NewMockExtractor

func NewMockExtractor(ctrl *gomock.Controller) *MockExtractor

NewMockExtractor creates a new mock instance.

func (*MockExtractor) EXPECT

func (m *MockExtractor) EXPECT() *MockExtractorMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockExtractor) Extract

func (m *MockExtractor) Extract(arg0 context.Context, arg1 ProcessFn) error

Extract mocks base method.

type MockExtractorMockRecorder

type MockExtractorMockRecorder struct {
	// contains filtered or unexported fields
}

MockExtractorMockRecorder is the mock recorder for MockExtractor.

func (*MockExtractorMockRecorder) Extract

func (mr *MockExtractorMockRecorder) Extract(arg0, arg1 interface{}) *gomock.Call

Extract indicates an expected call of Extract.

type MockScraper

type MockScraper struct {
	// contains filtered or unexported fields
}

MockScraper is a mock of Scraper interface.

func NewMockScraper

func NewMockScraper(ctrl *gomock.Controller) *MockScraper

NewMockScraper creates a new mock instance.

func (*MockScraper) Close

func (m *MockScraper) Close() error

Close mocks base method.

func (*MockScraper) EXPECT

func (m *MockScraper) EXPECT() *MockScraperMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockScraper) Scrape

func (m *MockScraper) Scrape(arg0 context.Context, arg1 []string, arg2 testkube.Execution) error

Scrape mocks base method.

type MockScraperMockRecorder

type MockScraperMockRecorder struct {
	// contains filtered or unexported fields
}

MockScraperMockRecorder is the mock recorder for MockScraper.

func (*MockScraperMockRecorder) Close

func (mr *MockScraperMockRecorder) Close() *gomock.Call

Close indicates an expected call of Close.

func (*MockScraperMockRecorder) Scrape

func (mr *MockScraperMockRecorder) Scrape(arg0, arg1, arg2 interface{}) *gomock.Call

Scrape indicates an expected call of Scrape.

type MockUploader

type MockUploader struct {
	// contains filtered or unexported fields
}

MockUploader is a mock of Uploader interface.

func NewMockUploader

func NewMockUploader(ctrl *gomock.Controller) *MockUploader

NewMockUploader creates a new mock instance.

func (*MockUploader) Close

func (m *MockUploader) Close() error

Close mocks base method.

func (*MockUploader) EXPECT

func (m *MockUploader) EXPECT() *MockUploaderMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockUploader) Upload

func (m *MockUploader) Upload(arg0 context.Context, arg1 *Object, arg2 testkube.Execution) error

Upload mocks base method.

type MockUploaderMockRecorder

type MockUploaderMockRecorder struct {
	// contains filtered or unexported fields
}

MockUploaderMockRecorder is the mock recorder for MockUploader.

func (*MockUploaderMockRecorder) Close

func (mr *MockUploaderMockRecorder) Close() *gomock.Call

Close indicates an expected call of Close.

func (*MockUploaderMockRecorder) Upload

func (mr *MockUploaderMockRecorder) Upload(arg0, arg1, arg2 interface{}) *gomock.Call

Upload indicates an expected call of Upload.

type Object

type Object struct {
	Name     string
	Size     int64
	Data     io.Reader
	DataType DataType
}

type ProcessFn

type ProcessFn func(ctx context.Context, object *Object) error

type RecursiveFilesystemExtractor

type RecursiveFilesystemExtractor struct {
	// contains filtered or unexported fields
}

func NewRecursiveFilesystemExtractor

func NewRecursiveFilesystemExtractor(fs filesystem.FileSystem) *RecursiveFilesystemExtractor

func (*RecursiveFilesystemExtractor) Extract

func (e *RecursiveFilesystemExtractor) Extract(ctx context.Context, paths []string, process ProcessFn) error

type Scraper

type Scraper interface {
	// Scrape gets artifacts from the provided paths and the provided execution
	Scrape(ctx context.Context, paths []string, execution testkube.Execution) error
	Close() error
}

Scraper is responsible for collecting and persisting the execution artifacts

type Uploader

type Uploader interface {
	Upload(ctx context.Context, object *Object, execution testkube.Execution) error
	Close() error
}