batchrouter

Documentation

Index

• func IsObjectStorageDestination(destType string) bool
• func IsWarehouseDestination(destType string) bool
• type AsyncPoll
• type AsyncStatusResponse
• type BatchedJobs
• type Connection
• type DestinationJobs
• type ErrorResponse
• type Factory
  • func (f *Factory) New(destType string) *Handle
• type Handle
  • func (brt *Handle) GetWorkspaceIDForDestID(destID string) string
  • func (brt *Handle) Setup(destType string, backendConfig backendconfig.BackendConfig, ...)
  • func (brt *Handle) Shutdown()
  • func (brt *Handle) Start()
• type JobParameters
• type ObjectStorageDefinition
• type UploadResult

Functions

func IsObjectStorageDestination

func IsObjectStorageDestination(destType string) bool

func IsWarehouseDestination

func IsWarehouseDestination(destType string) bool

Types

type AsyncPoll

type AsyncPoll struct {
	Config   map[string]interface{} `json:"config"`
	ImportId string                 `json:"importId"`
	DestType string                 `json:"destType"`
}

type AsyncStatusResponse

type AsyncStatusResponse struct {
	Success        bool
	StatusCode     int
	HasFailed      bool
	HasWarning     bool
	FailedJobsURL  string
	WarningJobsURL string
}

type BatchedJobs

type BatchedJobs struct {
	Jobs       []*jobsdb.JobT
	Connection *Connection
	TimeWindow time.Time
}

type Connection

type Connection struct {
	Source      backendconfig.SourceT
	Destination backendconfig.DestinationT
}

type DestinationJobs

type DestinationJobs struct {
	// contains filtered or unexported fields
}

type ErrorResponse

type ErrorResponse struct {
	Error string
}

type Factory

type Factory struct {
	Reporting        types.Reporting
	Multitenant      multitenant.MultiTenantI
	BackendConfig    backendconfig.BackendConfig
	RouterDB         jobsdb.JobsDB
	ProcErrorDB      jobsdb.JobsDB
	TransientSources transientsource.Service
	RsourcesService  rsources.JobService
	Debugger         destinationdebugger.DestinationDebugger
	AdaptiveLimit    func(int64) int64
}

func (*Factory) New

func (f *Factory) New(destType string) *Handle

type Handle

type Handle struct {
	Diagnostics diagnostics.DiagnosticsI
	// contains filtered or unexported fields
}

func (*Handle) GetWorkspaceIDForDestID

func (brt *Handle) GetWorkspaceIDForDestID(destID string) string

func (*Handle) Setup

func (brt *Handle) Setup(
	destType string,
	backendConfig backendconfig.BackendConfig,
	jobsDB, errorDB jobsdb.JobsDB,
	reporting types.Reporting,
	multitenantStat multitenant.MultiTenantI,
	transientSources transientsource.Service,
	rsourcesService rsources.JobService,
	debugger destinationdebugger.DestinationDebugger,
)

Setup initializes the batch router

func (*Handle) Shutdown

func (brt *Handle) Shutdown()

Shutdown stops the batch router

func (*Handle) Start

func (brt *Handle) Start()

Start starts the batch router's main loop

type JobParameters

type JobParameters struct {
	SourceID                string `json:"source_id"`
	DestinationID           string `json:"destination_id"`
	ReceivedAt              string `json:"received_at"`
	TransformAt             string `json:"transform_at"`
	SourceTaskRunID         string `json:"source_task_run_id"`
	SourceJobID             string `json:"source_job_id"`
	SourceJobRunID          string `json:"source_job_run_id"`
	SourceDefinitionID      string `json:"source_definition_id"`
	DestinationDefinitionID string `json:"destination_definition_id"`
	SourceCategory          string `json:"source_category"`
	EventName               string `json:"event_name"`
	EventType               string `json:"event_type"`
	MessageID               string `json:"message_id"`
}

type ObjectStorageDefinition

type ObjectStorageDefinition struct {
	Config          map[string]interface{}
	Key             string
	Provider        string
	DestinationID   string
	DestinationType string
}

type UploadResult

type UploadResult struct {
	Config           map[string]interface{}
	Key              string
	FileLocation     string
	LocalFilePaths   []string
	JournalOpID      int64
	Error            error
	FirstEventAt     string
	LastEventAt      string
	TotalEvents      int
	TotalBytes       int
	UseRudderStorage bool
}