rsources

package
v1.6.0-preview.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 8, 2023 License: AGPL-3.0 Imports: 17 Imported by: 0

Documentation

Overview

Package rsources is a generated GoMock package.

Index

Constants

This section is empty.

Variables

View Source 
var ErrOperationNotSupported = errors.New("rsources: operation not supported")

ErrOperationNotSupported sentinel error indicating an unsupported operation

View Source 
var ErrSourceNotCompleted = errors.New("Source not completed")

ErrSourceNotCompleted sentinel error indicating that a source is not completed

View Source 
var ErrStatusNotFound = errors.New("Status not found")

ErrStatusNotFound sentinel error indicating that status cannot be found

Functions

This section is empty.

Types

type DestinationFailedRecords added in v1.1.0 

type DestinationFailedRecords struct {
	ID      string        `json:"id"`
	Records FailedRecords `json:"records"`
}

type DestinationStatus 

type DestinationStatus struct {
	ID        string `json:"id"`
	Completed bool   `json:"completed"`
	Stats     Stats  `json:"stats"`
}

type FailedJobsStatsCollector added in v0.2.0 

type FailedJobsStatsCollector interface {
	StatsPublisher
	JobsFailed(jobs []*jobsdb.JobT)
}

FailedJobsStatsCollector collects stats for failed jobs

func NewFailedJobsCollector added in v0.2.0 

func NewFailedJobsCollector(jobservice JobService) FailedJobsStatsCollector

NewFailedJobsCollector creates a new stats collector for publishing failed job stats and records

type FailedRecords 

type FailedRecords []json.RawMessage

type Gauger added in v1.2.0 

type Gauger interface {
	Gauge(interface{})
}

type JobFailedRecords added in v1.1.0 

type JobFailedRecords struct {
	ID    string              `json:"id"`
	Tasks []TaskFailedRecords `json:"tasks"`
}

type JobFilter 

type JobFilter struct {
	TaskRunID []string
	SourceID  []string
}

type JobService 

type JobService interface {
	StatsIncrementer

	// Delete deletes all relevant information for a given jobRunId
	Delete(ctx context.Context, jobRunId string, filter JobFilter) error

	// GetStatus gets the current status of a job
	GetStatus(ctx context.Context, jobRunId string, filter JobFilter) (JobStatus, error)

	// AddFailedRecords adds failed records to the database as part of a transaction
	AddFailedRecords(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, records []json.RawMessage) error

	// GetFailedRecords gets the failed records for a jobRunID, with filters on taskRunId and sourceId
	GetFailedRecords(ctx context.Context, jobRunId string, filter JobFilter) (JobFailedRecords, error)

	// CleanupLoop starts the cleanup loop in the background which will stop upon context termination or in case of an error
	CleanupLoop(ctx context.Context) error

	// Monitor monitors the logical replication slot and lag when a shared database is configured
	Monitor(ctx context.Context, lagGauge, replicationSlotGauge Gauger)
}

JobService manages information about jobs created by rudder-sources

func NewJobService 

func NewJobService(config JobServiceConfig) (JobService, error)

func NewNoOpService 

func NewNoOpService() JobService

type JobServiceConfig added in v0.1.11 

type JobServiceConfig struct {
	LocalHostname               string
	LocalConn                   string
	MaxPoolSize                 int
	SharedConn                  string
	SubscriptionTargetConn      string
	SkipFailedRecordsCollection bool
	Log                         logger.Logger
}

type JobStatus 

type JobStatus struct {
	ID          string       `json:"id"`
	TasksStatus []TaskStatus `json:"tasks"`
}

type JobTargetKey 

type JobTargetKey struct {
	TaskRunID     string `json:"source_task_run_id"`
	SourceID      string `json:"source_id"`
	DestinationID string `json:"destination_id"`
}

func (JobTargetKey) String added in v1.1.0 

func (k JobTargetKey) String() string

type MockGauger added in v1.2.0 

type MockGauger struct {
	// contains filtered or unexported fields
}

MockGauger is a mock of Gauger interface.

func NewMockGauger added in v1.2.0 

func NewMockGauger(ctrl *gomock.Controller) *MockGauger

NewMockGauger creates a new mock instance.

func (*MockGauger) EXPECT added in v1.2.0 

func (m *MockGauger) EXPECT() *MockGaugerMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockGauger) Gauge added in v1.2.0 

func (m *MockGauger) Gauge(arg0 interface{})

Gauge mocks base method.

type MockGaugerMockRecorder added in v1.2.0 

type MockGaugerMockRecorder struct {
	// contains filtered or unexported fields
}

MockGaugerMockRecorder is the mock recorder for MockGauger.

func (*MockGaugerMockRecorder) Gauge added in v1.2.0 

func (mr *MockGaugerMockRecorder) Gauge(arg0 interface{}) *gomock.Call

Gauge indicates an expected call of Gauge.

type MockJobService 

type MockJobService struct {
	// contains filtered or unexported fields
}

MockJobService is a mock of JobService interface.

func NewMockJobService 

func NewMockJobService(ctrl *gomock.Controller) *MockJobService

NewMockJobService creates a new mock instance.

func (*MockJobService) AddFailedRecords 

func (m *MockJobService) AddFailedRecords(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, records []json.RawMessage) error

AddFailedRecords mocks base method.

func (*MockJobService) CleanupLoop 

func (m *MockJobService) CleanupLoop(ctx context.Context) error

CleanupLoop mocks base method.

func (*MockJobService) Delete 

func (m *MockJobService) Delete(ctx context.Context, jobRunId string, filter JobFilter) error

Delete mocks base method.

func (*MockJobService) EXPECT 

func (m *MockJobService) EXPECT() *MockJobServiceMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockJobService) GetFailedRecords 

func (m *MockJobService) GetFailedRecords(ctx context.Context, jobRunId string, filter JobFilter) (JobFailedRecords, error)

GetFailedRecords mocks base method.

func (*MockJobService) GetStatus 

func (m *MockJobService) GetStatus(ctx context.Context, jobRunId string, filter JobFilter) (JobStatus, error)

GetStatus mocks base method.

func (*MockJobService) IncrementStats 

func (m *MockJobService) IncrementStats(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, stats Stats) error

IncrementStats mocks base method.

func (*MockJobService) Monitor added in v1.2.0 

func (m *MockJobService) Monitor(ctx context.Context, lagGauge, replicationSlotGauge Gauger)

Monitor mocks base method.

type MockJobServiceMockRecorder 

type MockJobServiceMockRecorder struct {
	// contains filtered or unexported fields
}

MockJobServiceMockRecorder is the mock recorder for MockJobService.

func (*MockJobServiceMockRecorder) AddFailedRecords 

func (mr *MockJobServiceMockRecorder) AddFailedRecords(ctx, tx, jobRunId, key, records interface{}) *gomock.Call

AddFailedRecords indicates an expected call of AddFailedRecords.

func (*MockJobServiceMockRecorder) CleanupLoop 

func (mr *MockJobServiceMockRecorder) CleanupLoop(ctx interface{}) *gomock.Call

CleanupLoop indicates an expected call of CleanupLoop.

func (*MockJobServiceMockRecorder) Delete 

func (mr *MockJobServiceMockRecorder) Delete(ctx, jobRunId, filter interface{}) *gomock.Call

Delete indicates an expected call of Delete.

func (*MockJobServiceMockRecorder) GetFailedRecords 

func (mr *MockJobServiceMockRecorder) GetFailedRecords(ctx, jobRunId, filter interface{}) *gomock.Call

GetFailedRecords indicates an expected call of GetFailedRecords.

func (*MockJobServiceMockRecorder) GetStatus 

func (mr *MockJobServiceMockRecorder) GetStatus(ctx, jobRunId, filter interface{}) *gomock.Call

GetStatus indicates an expected call of GetStatus.

func (*MockJobServiceMockRecorder) IncrementStats 

func (mr *MockJobServiceMockRecorder) IncrementStats(ctx, tx, jobRunId, key, stats interface{}) *gomock.Call

IncrementStats indicates an expected call of IncrementStats.

func (*MockJobServiceMockRecorder) Monitor added in v1.2.0 

func (mr *MockJobServiceMockRecorder) Monitor(ctx, lagGauge, replicationSlotGauge interface{}) *gomock.Call

Monitor indicates an expected call of Monitor.

type MockStatsIncrementer 

type MockStatsIncrementer struct {
	// contains filtered or unexported fields
}

MockStatsIncrementer is a mock of StatsIncrementer interface.

func NewMockStatsIncrementer 

func NewMockStatsIncrementer(ctrl *gomock.Controller) *MockStatsIncrementer

NewMockStatsIncrementer creates a new mock instance.

func (*MockStatsIncrementer) EXPECT 

func (m *MockStatsIncrementer) EXPECT() *MockStatsIncrementerMockRecorder

EXPECT returns an object that allows the caller to indicate expected use.

func (*MockStatsIncrementer) IncrementStats 

func (m *MockStatsIncrementer) IncrementStats(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, stats Stats) error

IncrementStats mocks base method.

type MockStatsIncrementerMockRecorder 

type MockStatsIncrementerMockRecorder struct {
	// contains filtered or unexported fields
}

MockStatsIncrementerMockRecorder is the mock recorder for MockStatsIncrementer.

func (*MockStatsIncrementerMockRecorder) IncrementStats 

func (mr *MockStatsIncrementerMockRecorder) IncrementStats(ctx, tx, jobRunId, key, stats interface{}) *gomock.Call

IncrementStats indicates an expected call of IncrementStats.

type SourceFailedRecords added in v1.1.0 

type SourceFailedRecords struct {
	ID           string                     `json:"id"`
	Records      FailedRecords              `json:"records"`
	Destinations []DestinationFailedRecords `json:"destinations"`
}

type SourceStatus 

type SourceStatus struct {
	ID                 string              `json:"id"`
	Completed          bool                `json:"completed"`
	Stats              Stats               `json:"stats"`
	DestinationsStatus []DestinationStatus `json:"destinations"`
}

type Stats 

type Stats struct {
	In     uint `json:"in"`
	Out    uint `json:"out"`
	Failed uint `json:"failed"`
}

type StatsCollector 

type StatsCollector interface {
	StatsPublisher
	// JobsStored captures incoming job statistics
	JobsStored(jobs []*jobsdb.JobT)

	// JobsStoredWithErrors captures incoming job statistics
	JobsStoredWithErrors(jobs []*jobsdb.JobT, failedJobs map[uuid.UUID]string)

	// BeginProcessing prepares the necessary indices in order to
	// be ready for capturing JobStatus statistics
	BeginProcessing(jobs []*jobsdb.JobT)

	// JobStatusesUpdated captures outgoing job statistics.
	// A call to BeginProcessing must precede a call to this method,
	// so that all necessary indices can be created, since a JobStatus
	// doesn't carry all necessary job metadata such as jobRunId, taskRunId, etc.
	JobStatusesUpdated(jobStatuses []*jobsdb.JobStatusT)
}

StatsCollector collects and publishes stats as jobs are being created, processed and their statuses are being updated.

func NewStatsCollector 

func NewStatsCollector(jobservice JobService) StatsCollector

NewStatsCollector creates a new stats collector

type StatsIncrementer 

type StatsIncrementer interface {
	// IncrementStats increments the existing statistic counters
	// for a specific job measurement.
	IncrementStats(ctx context.Context, tx *sql.Tx, jobRunId string, key JobTargetKey, stats Stats) error
}

StatsIncrementer increments stats

type StatsPublisher added in v0.2.0 

type StatsPublisher interface {
	// Publish publishes statistics
	Publish(ctx context.Context, tx *sql.Tx) error
}

StatsPublisher publishes stats

type TaskFailedRecords added in v1.1.0 

type TaskFailedRecords struct {
	ID      string                `json:"id"`
	Sources []SourceFailedRecords `json:"sources"`
}

type TaskStatus 

type TaskStatus struct {
	ID            string         `json:"id"`
	SourcesStatus []SourceStatus `json:"sources"`
}

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.