batchrouter

Documentation

Index

• Variables
• func GetFullPrefix(manager filemanager.FileManager, prefix string) (fullPrefix string)
• func GetStorageDateFormat(manager filemanager.FileManager, destination *DestinationT, folderName string) (dateFormat string, err error)
• func Init()
• func IsAsyncDestination(destType string) bool
• func IsObjectStorageDestination(destType string) bool
• func IsWarehouseDestination(destType string) bool
• type AsyncPollT
• type AsyncStatusResponse
• type BatchDestinationDataT
• type BatchJobsT
• type DestinationT
• type ErrorResponseT
• type Factory
  • func (f *Factory) New(destType string) *HandleT
• type HandleT
  • func (brt *HandleT) Disable()
  • func (brt *HandleT) Enable()
  • func (brt *HandleT) GetWorkspaceIDForDestID(destID string) string
  • func (brt *HandleT) Setup(backendConfig backendconfig.BackendConfig, jobsDB, errorDB jobsdb.JobsDB, ...)
  • func (brt *HandleT) Shutdown()
  • func (brt *HandleT) Start()
• type JobParametersT
• type ObjectStorageT
• type StorageUploadOutput

Variables

var (
	Diagnostics diagnostics.DiagnosticsI
)

Functions

func GetFullPrefix

func GetFullPrefix(manager filemanager.FileManager, prefix string) (fullPrefix string)

func GetStorageDateFormat

func GetStorageDateFormat(manager filemanager.FileManager, destination *DestinationT, folderName string) (dateFormat string, err error)

func Init

func Init()

func IsAsyncDestination

func IsAsyncDestination(destType string) bool

func IsObjectStorageDestination

func IsObjectStorageDestination(destType string) bool

func IsWarehouseDestination

func IsWarehouseDestination(destType string) bool

Types

type AsyncPollT

type AsyncPollT struct {
	Config   map[string]interface{} `json:"config"`
	ImportId string                 `json:"importId"`
	DestType string                 `json:"destType"`
}

type AsyncStatusResponse

type AsyncStatusResponse struct {
	Success        bool
	StatusCode     int
	HasFailed      bool
	HasWarning     bool
	FailedJobsURL  string
	WarningJobsURL string
}

type BatchDestinationDataT

type BatchDestinationDataT struct {
	// contains filtered or unexported fields
}

type BatchJobsT

type BatchJobsT struct {
	Jobs             []*jobsdb.JobT
	BatchDestination *DestinationT
	TimeWindow       time.Time
}

type DestinationT

type DestinationT struct {
	Source      backendconfig.SourceT
	Destination backendconfig.DestinationT
}

type ErrorResponseT

type ErrorResponseT struct {
	Error string
}

type Factory

type Factory struct {
	Reporting        types.ReportingI
	Multitenant      multitenant.MultiTenantI
	BackendConfig    backendconfig.BackendConfig
	RouterDB         jobsdb.JobsDB
	ProcErrorDB      jobsdb.JobsDB
	TransientSources transientsource.Service
	RsourcesService  rsources.JobService
	Debugger         destinationdebugger.DestinationDebugger
	AdaptiveLimit    func(int64) int64
}

func (*Factory) New

func (f *Factory) New(destType string) *HandleT

type HandleT

type HandleT struct {
	// contains filtered or unexported fields
}

func (*HandleT) Disable

func (brt *HandleT) Disable()

Disable disables a router:)

func (*HandleT) Enable

func (brt *HandleT) Enable()

Enable enables a router :)

func (*HandleT) GetWorkspaceIDForDestID

func (brt *HandleT) GetWorkspaceIDForDestID(destID string) string

func (*HandleT) Setup

func (brt *HandleT) Setup(
	backendConfig backendconfig.BackendConfig,
	jobsDB,
	errorDB jobsdb.JobsDB,
	destType string,
	reporting types.ReportingI,
	multitenantStat multitenant.MultiTenantI,
	transientSources transientsource.Service,
	rsourcesService rsources.JobService,
	debugger destinationdebugger.DestinationDebugger,
)

Setup initializes this module

func (*HandleT) Shutdown

func (brt *HandleT) Shutdown()

func (*HandleT) Start

func (brt *HandleT) Start()

type JobParametersT

type JobParametersT struct {
	SourceID                string `json:"source_id"`
	DestinationID           string `json:"destination_id"`
	ReceivedAt              string `json:"received_at"`
	TransformAt             string `json:"transform_at"`
	SourceTaskRunID         string `json:"source_task_run_id"`
	SourceJobID             string `json:"source_job_id"`
	SourceJobRunID          string `json:"source_job_run_id"`
	SourceDefinitionID      string `json:"source_definition_id"`
	DestinationDefinitionID string `json:"destination_definition_id"`
	SourceCategory          string `json:"source_category"`
	EventName               string `json:"event_name"`
	EventType               string `json:"event_type"`
	MessageID               string `json:"message_id"`
}

JobParametersT struct holds source id and destination id of a job

type ObjectStorageT

type ObjectStorageT struct {
	Config          map[string]interface{}
	Key             string
	Provider        string
	DestinationID   string
	DestinationType string
}

type StorageUploadOutput

type StorageUploadOutput struct {
	Config           map[string]interface{}
	Key              string
	FileLocation     string
	LocalFilePaths   []string
	JournalOpID      int64
	Error            error
	FirstEventAt     string
	LastEventAt      string
	TotalEvents      int
	TotalBytes       int
	UseRudderStorage bool
}