batchrouter

Documentation

Index

• Variables
• func GetFullPrefix(manager filemanager.FileManager, prefix string) (fullPrefix string)
• func GetStorageDateFormat(manager filemanager.FileManager, destination *DestinationT, folderName string) (dateFormat string, err error)
• func Init()
• func Init2()
• func IsAsyncDestination(destType string) bool
• func IsObjectStorageDestination(destType string) bool
• func IsWarehouseDestination(destType string) bool
• type AsyncPollT
• type AsyncStatusResponse
• type BatchDestinationDataT
• type BatchJobsT
• type BatchRouterAdmin
  • func (ba *BatchRouterAdmin) Status() interface{}
• type DestinationT
• type ErrorResponseT
• type Factory
  • func (f *Factory) New(destType string) *HandleT
• type HandleT
  • func (brt *HandleT) Disable()
  • func (brt *HandleT) Enable()
  • func (brt *HandleT) GetWorkspaceIDForDestID(destID string) string
  • func (brt *HandleT) Setup(backendConfig backendconfig.BackendConfig, jobsDB, errorDB jobsdb.JobsDB, ...)
  • func (brt *HandleT) Shutdown()
  • func (brt *HandleT) Start()
• type JobParametersT
• type ObjectStorageT
• type StorageUploadOutput

Variables

var (
	Diagnostics  diagnostics.DiagnosticsI
	QueryFilters jobsdb.QueryFiltersT
)

Functions

func GetFullPrefix

func GetFullPrefix(manager filemanager.FileManager, prefix string) (fullPrefix string)

func GetStorageDateFormat

func GetStorageDateFormat(manager filemanager.FileManager, destination *DestinationT, folderName string) (dateFormat string, err error)

func Init

func Init()

func Init2

func Init2()

func IsAsyncDestination

func IsAsyncDestination(destType string) bool

func IsObjectStorageDestination

func IsObjectStorageDestination(destType string) bool

func IsWarehouseDestination

func IsWarehouseDestination(destType string) bool

Types

type AsyncPollT

type AsyncPollT struct {
	Config   map[string]interface{} `json:"config"`
	ImportId string                 `json:"importId"`
	DestType string                 `json:"destType"`
}

type AsyncStatusResponse

type AsyncStatusResponse struct {
	Success        bool
	StatusCode     int
	HasFailed      bool
	HasWarning     bool
	FailedJobsURL  string
	WarningJobsURL string
}

type BatchDestinationDataT

type BatchDestinationDataT struct {
	// contains filtered or unexported fields
}

type BatchJobsT

type BatchJobsT struct {
	Jobs             []*jobsdb.JobT
	BatchDestination *DestinationT
	TimeWindow       time.Time
}

type BatchRouterAdmin

type BatchRouterAdmin struct {
	// contains filtered or unexported fields
}

func (*BatchRouterAdmin) Status

func (ba *BatchRouterAdmin) Status() interface{}

Status function is used for debug purposes by the admin interface

type DestinationT

type DestinationT struct {
	Source      backendconfig.SourceT
	Destination backendconfig.DestinationT
}

type ErrorResponseT

type ErrorResponseT struct {
	Error string
}

type Factory

type Factory struct {
	Reporting        types.ReportingI
	Multitenant      multitenant.MultiTenantI
	BackendConfig    backendconfig.BackendConfig
	RouterDB         jobsdb.JobsDB
	ProcErrorDB      jobsdb.JobsDB
	TransientSources transientsource.Service
	RsourcesService  rsources.JobService
}

func (*Factory) New

func (f *Factory) New(destType string) *HandleT

type HandleT

type HandleT struct {
	// contains filtered or unexported fields
}

func (*HandleT) Disable

func (brt *HandleT) Disable()

Disable disables a router:)

func (*HandleT) Enable

func (brt *HandleT) Enable()

Enable enables a router :)

func (*HandleT) GetWorkspaceIDForDestID

func (brt *HandleT) GetWorkspaceIDForDestID(destID string) string

func (*HandleT) Setup

func (brt *HandleT) Setup(backendConfig backendconfig.BackendConfig, jobsDB, errorDB jobsdb.JobsDB, destType string, reporting types.ReportingI, multitenantStat multitenant.MultiTenantI, transientSources transientsource.Service, rsourcesService rsources.JobService)

Setup initializes this module

func (*HandleT) Shutdown

func (brt *HandleT) Shutdown()

func (*HandleT) Start

func (brt *HandleT) Start()

type JobParametersT

type JobParametersT struct {
	SourceID                string `json:"source_id"`
	DestinationID           string `json:"destination_id"`
	ReceivedAt              string `json:"received_at"`
	TransformAt             string `json:"transform_at"`
	SourceBatchID           string `json:"source_batch_id"`
	SourceTaskID            string `json:"source_task_id"`
	SourceTaskRunID         string `json:"source_task_run_id"`
	SourceJobID             string `json:"source_job_id"`
	SourceJobRunID          string `json:"source_job_run_id"`
	SourceDefinitionID      string `json:"source_definition_id"`
	DestinationDefinitionID string `json:"destination_definition_id"`
	SourceCategory          string `json:"source_category"`
	EventName               string `json:"event_name"`
	EventType               string `json:"event_type"`
	MessageID               string `json:"message_id"`
}

JobParametersT struct holds source id and destination id of a job

type ObjectStorageT

type ObjectStorageT struct {
	Config          map[string]interface{}
	Key             string
	Provider        string
	DestinationID   string
	DestinationType string
}

type StorageUploadOutput

type StorageUploadOutput struct {
	Config           map[string]interface{}
	Key              string
	FileLocation     string
	LocalFilePaths   []string
	JournalOpID      int64
	Error            error
	FirstEventAt     string
	LastEventAt      string
	TotalEvents      int
	UseRudderStorage bool
}