config

package
v0.0.0-...-73a5b24 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 30, 2024 License: Apache-2.0 Imports: 20 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func NewTunnelConfigLoader 

func NewTunnelConfigLoader()

Types

type ClickhouseSinkConfig 

type ClickhouseSinkConfig struct {
	Address           string                  `json:"address"`
	Username          string                  `json:"username"`
	Password          string                  `json:"password"`
	Database          string                  `json:"database"`
	TableName         string                  `json:"table_name"`
	BulkSize          int                     `json:"bulk_size"`
	IsAutoCreateTable bool                    `json:"is_auto_create_table"`
	Columns           []ClickhouseTableColumn `json:"columns"`
	Engine            string                  `json:"engine"`
	Partition         []string                `json:"partition"`
	PrimaryKey        []string                `json:"primary_key"`
	OrderBy           []string                `json:"order_by"`
	TTL               string                  `json:"ttl"`
	Comment           string                  `json:"comment"`
	Settings          []string                `json:"settings"`
}

ClickhouseSinkConfig Clickhouse Sink config

type ClickhouseTableColumn 

type ClickhouseTableColumn struct {
	Name     string `json:"name"`
	Type     string `json:"type"`
	Compress string `json:"compress"`
	Comment  string `json:"comment"`
}

ClickhouseTableColumn Clickhouse table column name

type ElasticsearchSinkConfig 

type ElasticsearchSinkConfig struct {
	Address   string `json:"address"`     // Elasticsearch address
	Username  string `json:"username"`    // Elasticsearch username
	Password  string `json:"password"`    // Elasticsearch password
	IndexName string `json:"index_name"`  // Elasticsearch index name
	DocIdName string `json:"doc_id_name"` // Elasticsearch document id name (will take it from transform data)
	Version   string `json:"version"`     // Only use 7.X or 8.X
}

ElasticsearchSinkConfig Elasticsearch Sink config

type HTTPSinkConfig 

type HTTPSinkConfig struct {
	URL                     string            `json:"url"`
	ContentType             string            `json:"content_type"`
	ReadTimeoutSecs         int               `json:"read_timeout_secs"`
	WriteTimeoutSecs        int               `json:"write_timeout_secs"`
	MaxIdleConnDurationSecs int               `json:"max_idle_conn_duration_secs"`
	MaxConnWaitTimeoutSecs  int               `json:"max_conn_wait_timeout_secs"`
	Headers                 map[string]string `json:"headers"`
}

HTTPSinkConfig HTTP Sink config

type KafkaSinkConfig 

type KafkaSinkConfig struct {
	Address     string `json:"address"`      // Kafka address
	Topic       string `json:"topic"`        // Kafka Topic
	MessageMode string `json:"message_mode"` // message mode: json, text
}

KafkaSinkConfig Kafka Sink config

type KafkaSourceConfig 

type KafkaSourceConfig struct {
	Address  string `json:"address"`  // kafka address
	Group    string `json:"group"`    // kafka group
	Topic    string `json:"topic"`    // kafka topic
	User     string `json:"user"`     // kafka user
	Password string `json:"password"` // kafka password
}

type LocalFileSinkConfig 

type LocalFileSinkConfig struct {
	FileName       string `json:"file_name"`
	FileFormatType string `json:"file_format_type"` // file format type: text, csv
	RowDelimiter   string `json:"row_delimiter"`    // only file_format_type = text is affect
}

LocalFileSinkConfig LocalFile Sink config

type MySQLSinkConfig 

type MySQLSinkConfig struct {
	Address   string `json:"address"`    // MySQL address
	Username  string `json:"username"`   // MySQL username
	Password  string `json:"password"`   // MySQL password
	Database  string `json:"database"`   // MySQL database
	TableName string `json:"table_name"` // MySQL table name
	BulkSize  int    `json:"bulk_size"`  // Bulk size
}

MySQLSinkConfig MySQL Sink config

type OracleSinkConfig 

type OracleSinkConfig struct {
	Address   string `json:"address"`    // Oracle address
	Username  string `json:"username"`   // Oracle username
	Password  string `json:"password"`   // Oracle password
	Database  string `json:"database"`   // Oracle database
	TableName string `json:"table_name"` // Oracle table name
	BulkSize  int    `json:"bulk_size"`  // Bulk size
}

OracleSinkConfig Oracle Sink config

type PostgresSQLSinkConfig 

type PostgresSQLSinkConfig struct {
	Address   string `json:"address"`    // PostgresSQL address
	Username  string `json:"username"`   // PostgresSQL username
	Password  string `json:"password"`   // PostgresSQL password
	Database  string `json:"database"`   // PostgresSQL database
	TableName string `json:"table_name"` // PostgresSQL table name
	BulkSize  int    `json:"bulk_size"`  // Bulk size
}

PostgresSQLSinkConfig PostgresSQL Sink config

type PromMetricsSourceConfig 

type PromMetricsSourceConfig struct {
	Address  string `json:"address"`  // prometheus address
	Interval int64  `json:"interval"` // prometheus interval
}

type PulsarSinkConfig 

type PulsarSinkConfig struct {
	Address     string `json:"address"`      // Pulsar host
	Topic       string `json:"topic"`        // Pulsar Topic
	MessageMode string `json:"message_mode"` // message mode: json, text
}

PulsarSinkConfig Pulsar Sink config

type PulsarSourceConfig 

type PulsarSourceConfig struct {
	Address          string `json:"address"`           // pulsar address
	Topic            string `json:"topic"`             // pulsar topic
	SubscriptionName string `json:"subscription_name"` // pulsar subscription name
}

type RabbitMQSinkConfig 

type RabbitMQSinkConfig struct {
	Address     string `json:"address"`      // RabbitMQ address
	Username    string `json:"username"`     // RabbitMQ username
	Password    string `json:"password"`     // RabbitMQ password
	VHost       string `json:"v_host"`       // RabbitMQ vhost
	Queue       string `json:"queue"`        // RabbitMQ queue
	Exchange    string `json:"exchange"`     // RabbitMQ exchange
	RoutingKey  string `json:"routing_key"`  // RabbitMQ routing key
	MessageMode string `json:"message_mode"` // message mode: json, text
}

RabbitMQSinkConfig RabbitMQ Sink config

type RabbitMQSourceConfig 

type RabbitMQSourceConfig struct {
	Address    string `json:"address"`     // rabbitmq address
	Username   string `json:"username"`    // rabbitmq username
	Password   string `json:"password"`    // rabbitmq password
	VHost      string `json:"v_host"`      // rabbitmq vhost
	Queue      string `json:"queue"`       // rabbitmq queue
	Exchange   string `json:"exchange"`    // rabbitmq exchange
	RoutingKey string `json:"routing_key"` // rabbitmq routing key
}

type RedisSinkConfig 

type RedisSinkConfig struct {
	DBNum            int    `json:"db_num"`              // db number（0-15）,cluster mode is invalid
	Expire           int    `json:"expire"`              // expire time, unit: seconds
	Address          string `json:"address"`             // redis hosts
	Username         string `json:"username"`            // Redis Username(6.0+)
	Password         string `json:"password"`            // Redis password
	DataType         string `json:"data_type"`           // data type: kv, hash, lpush, rpush, set, publish
	KeyOrChannelName string `json:"key_or_channel_name"` // Key or Channel name, if you want key is variable, please use "fromTransformHead"
}

RedisSinkConfig Redis Sink config

type RedisSourceConfig 

type RedisSourceConfig struct {
	DBNum            int    `json:"db_num"`              // db number（0-15）,cluster mode is invalid
	KeyOrChannelName string `json:"key_or_channel_name"` // Key or Channel name
	Address          string `json:"address"`             // redis hosts
	Username         string `json:"username"`            // Redis Username(6.0+)
	Password         string `json:"password"`            // Redis password
	DataType         string `json:"data_type"`           // data type: lpop, rpop, subscribe
}

type RocketMQSinkConfig 

type RocketMQSinkConfig struct {
	Address     string `json:"address"`      // RocketMQ address
	Topic       string `json:"topic"`        // RocketMQ Topic
	MessageMode string `json:"message_mode"` // message mode: json, text
}

RocketMQSinkConfig RocketMQ Sink config

type RocketMQSourceConfig 

type RocketMQSourceConfig struct {
	Address   string `json:"address"`    // rocketmq address
	Group     string `json:"group"`      // rocketmq group
	Topic     string `json:"topic"`      // rocketmq topic
	AccessKey string `json:"access_key"` // rocketmq access key
	SecretKey string `json:"secret_key"` // rocketmq secret key
}

type SinkConfig 

type SinkConfig struct {
	Type          string                  `json:"type" vd:"len($)>0"`
	SinkName      string                  `json:"sink_name" vd:"len($)>0"`
	Clickhouse    ClickhouseSinkConfig    `json:"clickhouse"`
	HTTP          HTTPSinkConfig          `json:"http"`
	Kafka         KafkaSinkConfig         `json:"kafka"`
	Redis         RedisSinkConfig         `json:"redis"`
	LocalFile     LocalFileSinkConfig     `json:"local_file"`
	PostgresSQL   PostgresSQLSinkConfig   `json:"postgres_sql"`
	RocketMQ      RocketMQSinkConfig      `json:"rocketmq"`
	RabbitMQ      RabbitMQSinkConfig      `json:"rabbitmq"`
	Oracle        OracleSinkConfig        `json:"oracle"`
	MySQL         MySQLSinkConfig         `json:"mysql"`
	Pulsar        PulsarSinkConfig        `json:"pulsar"`
	Elasticsearch ElasticsearchSinkConfig `json:"elasticsearch"`
}

SinkConfig Sink config

type SourceConfig 

type SourceConfig struct {
	Type        string                  `json:"type" vd:"len($)>0"`
	SourceName  string                  `json:"source_name" vd:"len($)>0"`
	Kafka       KafkaSourceConfig       `json:"kafka"`
	RocketMQ    RocketMQSourceConfig    `json:"rocketmq"`
	RabbitMQ    RabbitMQSourceConfig    `json:"rabbitmq"`
	PromMetrics PromMetricsSourceConfig `json:"prom_metrics"`
	Pulsar      PulsarSourceConfig      `json:"pulsar"`
	Redis       RedisSourceConfig       `json:"redis"`
}

SourceConfig Source config

type StreamConfig 

type StreamConfig struct {
	Name          string          `json:"name" vd:"len($)>0"`   // stream name
	Enable        bool            `json:"enable"`               // is enabled stream
	ChannelSize   int             `json:"channel_size"`         // channel buffer size（default: 0）
	Source        []*SourceConfig `json:"source" vd:"len($)>0"` // source config slice
	SourceAckMode int             `json:"ack_mode"`             // It worked only Source is MQ component; 0: commit after consume; 1: commit after transform; 2: commit after sink
	Transform     TransformConfig `json:"transform"`            // transform config
	Sink          []*SinkConfig   `json:"sink" vd:"len($)>0"`   // sink config slice
}

StreamConfig stream config

func (*StreamConfig) GetSinkBySinkName 

func (s *StreamConfig) GetSinkBySinkName(name string) *SinkConfig

GetSinkBySinkName based on SinkName to get Sink

func (*StreamConfig) GetSinkTagBySinkName 

func (s *StreamConfig) GetSinkTagBySinkName(name string) string

GetSinkTagBySinkName based on SinkName to get Sink Tag

func (*StreamConfig) GetSourceBySourceName 

func (s *StreamConfig) GetSourceBySourceName(name string) *SourceConfig

GetSourceBySourceName based on SourceName to get Source

func (*StreamConfig) GetSourceTagBySourceName 

func (s *StreamConfig) GetSourceTagBySourceName(name string) string

GetSourceTagBySourceName based on SourceName to get Source Tag

type TransformConfig 

type TransformConfig struct {
	Mode         string              `json:"mode" vd:"len($)>0"` // Mode: row, json, jsonPath
	Schemas      []TransformSchema   `json:"schemas"`            // Schema
	RowSeparator string              `json:"row_separator"`      // only mode is row will affect, and also only row mode will use strings.Split to split
	Paths        []TransformJsonPath `json:"paths"`              // json paths
}

TransformConfig transform config

type TransformJsonPath 

type TransformJsonPath struct {
	SrcField  string `json:"src_field"`  // source field
	Path      string `json:"path"`       // json path
	DestField string `json:"dest_field"` // destination field
}

TransformJsonPath transform json path config

type TransformSchema 

type TransformSchema struct {
	SourceKey    string `json:"source_key" vd:"len($)>0"`                       // source key
	SinkKey      string `json:"sink_key" vd:"len($)>0 || (IsIgnore)$ == true"`  // sink key
	Converter    string `json:"converter"`                                      // Converter, Like: toInt, toFloat32, toString, etc.
	IsIgnore     bool   `json:"is_ignore"`                                      // is ignored key
	IsStrictMode bool   `json:"is_strict_mode"`                                 // is strict mode
	IsKeepKeys   bool   `json:"is_keep_keys"`                                   // key is keep origin key
	IsExpand     bool   `json:"is_expand"`                                      // is expanded col
	ExpandValue  any    `json:"expand_value"`                                   // expand value
	SourceName   string `json:"source_name" vd:"len($)>0"`                      // source alias name
	SinkName     string `json:"sink_name" vd:"len($)>0 || (IsIgnore)$ == true"` // sink alias name
}

TransformSchema transform unit config when is_ignore is True, sink_key and sink_name can be empty.

type TunnelConfig 

type TunnelConfig struct {
	Streams []*StreamConfig `json:"streams" vd:"len($)>0"`
}

TunnelConfig data tunnel config struct. 

var TunnelCfg *TunnelConfig

TunnelCfg data tunnel config.

type TunnelConfigLoader 

type TunnelConfigLoader struct {
	// contains filtered or unexported fields
}

func (*TunnelConfigLoader) IsConfigLoaded 

func (c *TunnelConfigLoader) IsConfigLoaded() bool

func (*TunnelConfigLoader) Load 

func (c *TunnelConfigLoader) Load()

Load load config api

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.