neosync_benthos

Documentation

Index

• type ArchiveProcessor
• type AwsCredentials
• type AwsS3Insert
• type Backoff
• type BatchProcessor
• type Batching
• type BenthosConfig
• type BufferConfig
• type CompressProcessor
• type DataStream
• type DropConfig
• type DropOnConfig
• type HTTPConfig
• type Headers
• type HttpClient
• type InlineRetryConfig
• type InputConfig
• type Inputs
• type NeosyncConnectionData
• type OutputBrokerConfig
• type OutputConfig
• type Outputs
• type PipelineConfig
• type ProcessorConfig
• type Protobuf
• type RetryConfig
• type SqlInsert
• type SqlRaw
• type SqlSelect
• type Stream
• type StreamConfig

Types

type ArchiveProcessor

type ArchiveProcessor struct {
	Format string  `json:"format" yaml:"format"`
	Path   *string `json:"path,omitempty" yaml:"path,omitempty"`
}

type AwsCredentials

type AwsCredentials struct {
	Profile        string `json:"profile,omitempty" yaml:"profile,omitempty"`
	Id             string `json:"id,omitempty" yaml:"id,omitempty"`
	Secret         string `json:"secret,omitempty" yaml:"secret,omitempty"`
	Token          string `json:"token,omitempty" yaml:"token,omitempty"`
	FromEc2Role    bool   `json:"from_ec2_role,omitempty" yaml:"from_ec2_role,omitempty"`
	Role           string `json:"role,omitempty" yaml:"role,omitempty"`
	RoleExternalId string `json:"role_external_id,omitempty" yaml:"role_external_id,omitempty"`
}

type AwsS3Insert

type AwsS3Insert struct {
	Bucket      string    `json:"bucket" yaml:"bucket"`
	MaxInFlight int       `json:"max_in_flight" yaml:"max_in_flight"`
	Path        string    `json:"path" yaml:"path"`
	Batching    *Batching `json:"batching,omitempty" yaml:"batching,omitempty"`

	Region   string `json:"region,omitempty" yaml:"region,omitempty"`
	Endpoint string `json:"endpoint,omitempty" yaml:"endpoint,omitempty"`

	Credentials *AwsCredentials `json:"credentials,omitempty" yaml:"credentials,omitempty"`
}

type Backoff

type Backoff struct {
	InitialInterval string `json:"initial_interval" yaml:"initial_interval"`
	MaxInterval     string `json:"max_interval" yaml:"max_interval"`
	MaxElapsedTime  string `json:"max_elapsed_time" yaml:"max_elapsed_time"`
}

type BatchProcessor

type BatchProcessor struct {
	Archive  *ArchiveProcessor  `json:"archive,omitempty" yaml:"archive,omitempty"`
	Compress *CompressProcessor `json:"compress,omitempty" yaml:"compress,omitempty"`
}

type Batching

type Batching struct {
	Count      int               `json:"count" yaml:"count"`
	ByteSize   int               `json:"byte_size" yaml:"byte_size"`
	Period     string            `json:"period" yaml:"period"`
	Check      string            `json:"check" yaml:"check"`
	Processors []*BatchProcessor `json:"processors" yaml:"processors"`
}

type BenthosConfig

type BenthosConfig struct {
	// HTTP         HTTPConfig `json:"http" yaml:"http"`
	StreamConfig `json:",inline" yaml:",inline"`
}

type BufferConfig

type BufferConfig struct{}

type CompressProcessor

type CompressProcessor struct {
	Algorithm string `json:"algorithm" yaml:"algorithm"`
}

type DataStream

type DataStream struct{}

type DropConfig

type DropConfig struct{}

type DropOnConfig

type DropOnConfig struct {
	Error        bool    `json:"error" yaml:"error"`
	Backpressure string  `json:"back_pressure" yaml:"back_pressure"`
	Output       Outputs `json:"output" yaml:"output"`
}

type HTTPConfig

type HTTPConfig struct {
	Address string `json:"address" yaml:"address"`
	Enabled bool   `json:"enabled" yaml:"enabled"`
}

type Headers

type Headers struct {
	Authorization *string `json:"Authorization,omitempty" yaml:"Authorization,omitempty"`
	ContentType   *string `json:"Content-Type,omitempty" yaml:"Content-Type,omitempty"`
	Accept        *string `json:"Accept,omitempty" yaml:"Accept,omitempty"`
}

type HttpClient

type HttpClient struct {
	Url       string   `json:"url" yaml:"url"`
	Verb      string   `json:"verb" yaml:"verb"`
	Headers   *Headers `json:"headers,omitempty" yaml:"headers,omitempty"`
	RateLimit *string  `json:"rate_limit,omitempty" yaml:"rate_limit,omitempty"`
	Timeout   string   `json:"timeout" yaml:"timeout"`
	Payload   *string  `json:"payload,omitempty" yaml:"payload,omitempty"`
	Stream    *Stream  `json:"stream,omitempty" yaml:"stream,omitempty"`
}

type InlineRetryConfig

type InlineRetryConfig struct {
	MaxRetries uint64  `json:"max_retries" yaml:"max_retries"`
	Backoff    Backoff `json:"backoff" yaml:"backoff"`
}

type InputConfig

type InputConfig struct {
	Label  string `json:"label" yaml:"label"`
	Inputs `json:",inline" yaml:",inline"`
}

type Inputs

type Inputs struct {
	SqlSelect             *SqlSelect             `json:"sql_select,omitempty" yaml:"sql_select,omitempty"`
	HttpClient            *HttpClient            `json:"http_client,omitempty" yaml:"http_client,omitempty"`
	NeosyncConnectionData *NeosyncConnectionData `json:"neosync_connection_data,omitempty" yaml:"neosync_connection_data,omitempty"`
}

type NeosyncConnectionData

type NeosyncConnectionData struct {
	ApiKey         *string `json:"api_key,omitempty" yaml:"api_key,omitempty"`
	ApiUrl         string  `json:"api_url" yaml:"api_url"`
	ConnectionId   string  `json:"connection_id" yaml:"connection_id"`
	ConnectionType string  `json:"connection_type" yaml:"connection_type"`
	JobId          *string `json:"job_id,omitempty" yaml:"job_id,omitempty"`
	JobRunId       *string `json:"job_run_id,omitempty" yaml:"job_run_id,omitempty"`
	Schema         string  `json:"schema" yaml:"schema"`
	Table          string  `json:"table" yaml:"table"`
}

type OutputBrokerConfig

type OutputBrokerConfig struct {
	Pattern string    `json:"pattern" yaml:"pattern"`
	Outputs []Outputs `json:"outputs" yaml:"outputs"`
}

type OutputConfig

type OutputConfig struct {
	Label   string `json:"label" yaml:"label"`
	Outputs `json:",inline" yaml:",inline"`
}

type Outputs

type Outputs struct {
	SqlInsert *SqlInsert          `json:"sql_insert,omitempty" yaml:"sql_insert,omitempty"`
	SqlRaw    *SqlRaw             `json:"sql_raw,omitempty" yaml:"sql_raw,omitempty"`
	AwsS3     *AwsS3Insert        `json:"aws_s3,omitempty" yaml:"aws_s3,omitempty"`
	Retry     *RetryConfig        `json:"retry,omitempty" yaml:"retry,omitempty"`
	Broker    *OutputBrokerConfig `json:"broker,omitempty" yaml:"broker,omitempty"`
	DropOn    *DropOnConfig       `json:"drop_on,omitempty" yaml:"drop_on,omitempty"`
	Drop      *DropConfig         `json:"drop,omitempty" yaml:"drop,omitempty"`
	Resource  string              `json:"resource,omitempty" yaml:"resource,omitempty"`
	Fallback  []Outputs           `json:"fallback,omitempty" yaml:"fallback,omitempty"`
}

type PipelineConfig

type PipelineConfig struct {
	Threads    int               `json:"threads" yaml:"threads"`
	Processors []ProcessorConfig `json:"processors" yaml:"processors"`
}

type ProcessorConfig

type ProcessorConfig struct {
	Mutation   string      `json:"mutation,omitempty" yaml:"mutation,omitempty"`
	Protobuf   *Protobuf   `json:"protobuf,omitempty" yaml:"protobuf,omitempty"`
	DataStream *DataStream `json:"datastream,omitempty" yaml:"datastream,omitempty"`
}

type Protobuf

type Protobuf struct {
	Operator    string   `json:"operator" yaml:"operator"`
	Message     string   `json:"message" yaml:"message"`
	ImportPaths []string `json:"import_paths" yaml:"import_paths"`
}

type RetryConfig

type RetryConfig struct {
	Output            OutputConfig `json:"output" yaml:"output"`
	InlineRetryConfig `json:",inline" yaml:",inline"`
}

type SqlInsert

type SqlInsert struct {
	Driver          string    `json:"driver" yaml:"driver"`
	Dsn             string    `json:"dsn" yaml:"dsn"`
	Table           string    `json:"table" yaml:"table"`
	Columns         []string  `json:"columns" yaml:"columns"`
	ArgsMapping     string    `json:"args_mapping" yaml:"args_mapping"`
	InitStatement   string    `json:"init_statement" yaml:"init_statement"`
	ConnMaxIdleTime string    `json:"conn_max_idle_time,omitempty" yaml:"conn_max_idle_time,omitempty"`
	ConnMaxLifeTime string    `json:"conn_max_life_time,omitempty" yaml:"conn_max_life_time,omitempty"`
	ConnMaxIdle     int       `json:"conn_max_idle,omitempty" yaml:"conn_max_idle,omitempty"`
	ConnMaxOpen     int       `json:"conn_max_open,omitempty" yaml:"conn_max_open,omitempty"`
	Batching        *Batching `json:"batching,omitempty" yaml:"batching,omitempty"`
}

type SqlRaw

type SqlRaw struct {
	Driver          string    `json:"driver" yaml:"driver"`
	Dsn             string    `json:"dsn" yaml:"dsn"`
	Query           string    `json:"query" yaml:"query"`
	ArgsMapping     string    `json:"args_mapping" yaml:"args_mapping"`
	InitStatement   string    `json:"init_statement" yaml:"init_statement"`
	ConnMaxIdleTime string    `json:"conn_max_idle_time,omitempty" yaml:"conn_max_idle_time,omitempty"`
	ConnMaxLifeTime string    `json:"conn_max_life_time,omitempty" yaml:"conn_max_life_time,omitempty"`
	ConnMaxIdle     int       `json:"conn_max_idle,omitempty" yaml:"conn_max_idle,omitempty"`
	ConnMaxOpen     int       `json:"conn_max_open,omitempty" yaml:"conn_max_open,omitempty"`
	Batching        *Batching `json:"batching,omitempty" yaml:"batching,omitempty"`
}

type SqlSelect

type SqlSelect struct {
	Driver        string   `json:"driver" yaml:"driver"`
	Dsn           string   `json:"dsn" yaml:"dsn"`
	Table         string   `json:"table" yaml:"table"`
	Columns       []string `json:"columns" yaml:"columns"`
	Where         string   `json:"where,omitempty" yaml:"where,omitempty"`
	ArgsMapping   string   `json:"args_mapping,omitempty" yaml:"args_mapping,omitempty"`
	InitStatement string   `json:"init_statement,omitempty" yaml:"init_statement,omitempty"`
}

type Stream

type Stream struct {
	Enabled   bool   `json:"enabled,omitempty" yaml:"enabled,omitempty"`
	Reconnect bool   `json:"reconnect,omitempty" yaml:"reconnect,omitempty"`
	Codec     string `json:"codec" yaml:"codec"`
}

type StreamConfig

type StreamConfig struct {
	Input    *InputConfig    `json:"input" yaml:"input"`
	Buffer   *BufferConfig   `json:"buffer,omitempty" yaml:"buffer,omitempty"`
	Pipeline *PipelineConfig `json:"pipeline,omitempty" yaml:"pipeline,omitempty"`
	Output   *OutputConfig   `json:"output" yaml:"output"`
}