rsources

package
v1.0.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 25, 2022 License: AGPL-3.0 Imports: 16 Imported by: 0

Documentation

Overview

Package rsources is a generated GoMock package.

Index

Constants

This section is empty.

Variables

View Source 
var ErrOperationNotSupported = errors.New("rsources: operation not supported")

ErrOperationNotSupported sentinel error indicating an unsupported operation

View Source 
var StatusNotFoundError = errors.New("Status not found")

Functions

This section is empty.

Types

type DestinationStatus 

type DestinationStatus struct {
	ID        string `json:"id"`
	Completed bool   `json:"completed"`
	Stats     Stats  `json:"stats"`
}

type FailedJobsStatsCollector added in v0.2.0 

type FailedJobsStatsCollector interface {
	StatsPublisher
	JobsFailed(jobs []*jobsdb.JobT)
}

FailedJobsStatsCollector collects stats for failed jobs

func NewFailedJobsCollector added in v0.2.0 

func NewFailedJobsCollector(jobservice JobService) FailedJobsStatsCollector

NewFailedJobsCollector creates a new stats collector for publishing failed job stats and records

type FailedRecord added in v0.2.0 

type FailedRecord struct {
	JobRunID      string          `json:"job_run_id"`
	TaskRunID     string          `json:"task_run_id"`
	SourceID      string          `json:"source_id"`
	DestinationID string          `json:"destination_id"`
	RecordID      json.RawMessage `json:"record_id"`
	CreatedAt     time.Time       `json:"-"`
}

type FailedRecords 

type FailedRecords []FailedRecord

type JobFilter 

type JobFilter struct {
	TaskRunID []string
	SourceID  []string
}

type JobService 

type JobService interface {
	StatsIncrementer

	// Delete deletes all relevant information for a given jobRunId
	Delete(ctx context.Context, jobRunId string) error

	// GetStatus gets the current status of a job
	GetStatus(ctx context.Context, jobRunId string, filter JobFilter) (JobStatus, error)

	// AddFailedRecords adds failed records to the database as part of a transaction
	AddFailedRecords(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, records []json.RawMessage) error

	// GetFailedRecords gets the failed records for a jobRunID, with filters on taskRunId and sourceId
	GetFailedRecords(ctx context.Context, jobRunId string, filter JobFilter) (FailedRecords, error)

	// CleanupLoop starts the cleanup loop in the background which will stop upon context termination or in case of an error
	CleanupLoop(ctx context.Context) error
}

JobService manages information about jobs created by rudder-sources

func NewJobService 

func NewJobService(config JobServiceConfig) (JobService, error)

func NewNoOpService 

func NewNoOpService() JobService

type JobServiceConfig added in v0.1.11 

type JobServiceConfig struct {
	LocalHostname               string
	LocalConn                   string
	MaxPoolSize                 int
	SharedConn                  string
	SubscriptionTargetConn      string
	SkipFailedRecordsCollection bool
}

type JobStatus 

type JobStatus struct {
	ID          string       `json:"id"`
	TasksStatus []TaskStatus `json:"tasks"`
}

type JobTargetKey 

type JobTargetKey struct {
	TaskRunID     string `json:"source_task_run_id"`
	SourceID      string `json:"source_id"`
	DestinationID string `json:"destination_id"`
}

type MockJobService 

type MockJobService struct {
	// contains filtered or unexported fields
}

MockJobService is a mock of JobService interface.

func NewMockJobService 

func NewMockJobService(ctrl *gomock.Controller) *MockJobService

NewMockJobService creates a new mock instance.

func (*MockJobService) AddFailedRecords 

func (m *MockJobService) AddFailedRecords(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, records []json.RawMessage) error

AddFailedRecords mocks base method.

func (*MockJobService) CleanupLoop 

func (m *MockJobService) CleanupLoop(ctx context.Context) error

CleanupLoop mocks base method.

func (*MockJobService) Delete 

func (m *MockJobService) Delete(ctx context.Context, jobRunId string) error

Delete mocks base method.

func (*MockJobService) EXPECT 

func (m *MockJobService) EXPECT() *MockJobServiceMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockJobService) GetFailedRecords 

func (m *MockJobService) GetFailedRecords(ctx context.Context, jobRunId string, filter JobFilter) (FailedRecords, error)

GetFailedRecords mocks base method.

func (*MockJobService) GetStatus 

func (m *MockJobService) GetStatus(ctx context.Context, jobRunId string, filter JobFilter) (JobStatus, error)

GetStatus mocks base method.

func (*MockJobService) IncrementStats 

func (m *MockJobService) IncrementStats(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, stats Stats) error

IncrementStats mocks base method.

type MockJobServiceMockRecorder 

type MockJobServiceMockRecorder struct {
	// contains filtered or unexported fields
}

MockJobServiceMockRecorder is the mock recorder for MockJobService.

func (*MockJobServiceMockRecorder) AddFailedRecords 

func (mr *MockJobServiceMockRecorder) AddFailedRecords(ctx, tx, jobRunId, key, records interface{}) *gomock.Call

AddFailedRecords indicates an expected call of AddFailedRecords.

func (*MockJobServiceMockRecorder) CleanupLoop 

func (mr *MockJobServiceMockRecorder) CleanupLoop(ctx interface{}) *gomock.Call

CleanupLoop indicates an expected call of CleanupLoop.

func (*MockJobServiceMockRecorder) Delete 

func (mr *MockJobServiceMockRecorder) Delete(ctx, jobRunId interface{}) *gomock.Call

Delete indicates an expected call of Delete.

func (*MockJobServiceMockRecorder) GetFailedRecords 

func (mr *MockJobServiceMockRecorder) GetFailedRecords(ctx, jobRunId, filter interface{}) *gomock.Call

GetFailedRecords indicates an expected call of GetFailedRecords.

func (*MockJobServiceMockRecorder) GetStatus 

func (mr *MockJobServiceMockRecorder) GetStatus(ctx, jobRunId, filter interface{}) *gomock.Call

GetStatus indicates an expected call of GetStatus.

func (*MockJobServiceMockRecorder) IncrementStats 

func (mr *MockJobServiceMockRecorder) IncrementStats(ctx, tx, jobRunId, key, stats interface{}) *gomock.Call

IncrementStats indicates an expected call of IncrementStats.

type MockStatsIncrementer 

type MockStatsIncrementer struct {
	// contains filtered or unexported fields
}

MockStatsIncrementer is a mock of StatsIncrementer interface.

func NewMockStatsIncrementer 

func NewMockStatsIncrementer(ctrl *gomock.Controller) *MockStatsIncrementer

NewMockStatsIncrementer creates a new mock instance.

func (*MockStatsIncrementer) EXPECT 

func (m *MockStatsIncrementer) EXPECT() *MockStatsIncrementerMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockStatsIncrementer) IncrementStats 

func (m *MockStatsIncrementer) IncrementStats(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, stats Stats) error

IncrementStats mocks base method.

type MockStatsIncrementerMockRecorder 

type MockStatsIncrementerMockRecorder struct {
	// contains filtered or unexported fields
}

MockStatsIncrementerMockRecorder is the mock recorder for MockStatsIncrementer.

func (*MockStatsIncrementerMockRecorder) IncrementStats 

func (mr *MockStatsIncrementerMockRecorder) IncrementStats(ctx, tx, jobRunId, key, stats interface{}) *gomock.Call

IncrementStats indicates an expected call of IncrementStats.

type SourceStatus 

type SourceStatus struct {
	ID                 string              `json:"id"`
	Completed          bool                `json:"completed"`
	Stats              Stats               `json:"stats"`
	DestinationsStatus []DestinationStatus `json:"destinations"`
}

type Stats 

type Stats struct {
	In     uint `json:"in"`
	Out    uint `json:"out"`
	Failed uint `json:"failed"`
}

type StatsCollector 

type StatsCollector interface {
	StatsPublisher
	// JobsStored captures incoming job statistics
	JobsStored(jobs []*jobsdb.JobT)

	// JobsStoredWithErrors captures incoming job statistics
	JobsStoredWithErrors(jobs []*jobsdb.JobT, failedJobs map[uuid.UUID]string)

	// BeginProcessing prepares the necessary indices in order to
	// be ready for capturing JobStatus statistics
	BeginProcessing(jobs []*jobsdb.JobT)

	// JobStatusesUpdated captures outgoing job statistics.
	// A call to BeginProcessing must precede a call to this method,
	// so that all necessary indices can be created, since a JobStatus
	// doesn't carry all necessary job metadata such as jobRunId, taskRunId, etc.
	JobStatusesUpdated(jobStatuses []*jobsdb.JobStatusT)
}

StatsCollector collects and publishes stats as jobs are being created, processed and their statuses are being updated.

func NewStatsCollector 

func NewStatsCollector(jobservice JobService) StatsCollector

NewStatsCollector creates a new stats collector

type StatsIncrementer 

type StatsIncrementer interface {
	// IncrementStats increments the existing statistic counters
	// for a specific job measurement.
	IncrementStats(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, stats Stats) error
}

StatsIncrementer increments stats

type StatsPublisher added in v0.2.0 

type StatsPublisher interface {
	// Publish publishes statistics
	Publish(ctx context.Context, tx *sql.Tx) error
}

StatsPublisher publishes stats

type TaskStatus 

type TaskStatus struct {
	ID            string         `json:"id"`
	SourcesStatus []SourceStatus `json:"sources"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.