spec

Documentation

Index

• Constants
• Variables
• type Engine
  • func DefaultEngine() *Engine
  • func (e *Engine) String() string
  • func (e *Engine) Validate() error
• type OrderByStrategy
• type PartitionStrategy
• type Spec
  • func (Spec) JSONSchemaExtend(sc *jsonschema.Schema)
  • func (s *Spec) Options() (*clickhouse.Options, error)
  • func (s *Spec) SetDefaults()
  • func (s *Spec) Validate() error

Constants

const (
	MergeTree = "MergeTree"
)

Variables

var JSONSchema string

Types

type Engine

type Engine struct {
	// Name of the table engine.
	// Only [`*MergeTree` family](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family) is supported at the moment.
	Name string `json:"name,omitempty" jsonschema:"pattern=^.*MergeTree$,default=MergeTree"`

	// Engine parameters.
	// Currently, no restrictions are imposed on the parameter types.
	Parameters []any `json:"parameters,omitempty"`
}

Engine allows to specify a custom table engine to be used.

func DefaultEngine

func DefaultEngine() *Engine

func (*Engine) String

func (e *Engine) String() string

func (*Engine) Validate

func (e *Engine) Validate() error

type OrderByStrategy

type OrderByStrategy struct {
	// Table glob patterns that apply for this ORDER BY clause.
	//
	// If unset, the ORDER BY clause will apply to all tables.
	//
	// If a table matches both a pattern in `tables` and `skip_tables`, the table will be skipped.
	//
	// Order by strategy table patterns should be disjointed sets: if a table matches two order by strategies,
	// an error will be raised at runtime.
	Tables []string `json:"tables,omitempty"`

	// Table glob patterns that should be skipped for this ORDER BY clause.
	//
	// If unset, no tables will be skipped.
	//
	// If a table matches both a pattern in `tables` and `skip_tables`, the table will be skipped.
	//
	// Order by strategy table patterns should be disjointed sets: if a table matches two order by strategies,
	// an error will be raised at runtime.
	SkipTables []string `json:"skip_tables,omitempty"`

	// ORDER BY list of expressions to use, e.g. `_cq_sync_group_id, toYYYYMM(_cq_sync_time), _cq_id`,
	// the strings are passed as is after "ORDER BY" clause, separated by commas, with no validation or quoting.
	//
	// An unset order_by is not valid.
	OrderBy []string `json:"order_by"`
}

type PartitionStrategy

type PartitionStrategy struct {
	// Table glob patterns that apply for this partitioning.
	//
	// If unset, the partitioning will apply to all tables.
	//
	// If a table matches both a pattern in `tables` and `skip_tables`, the table will be skipped.
	//
	// Partition strategy table patterns should be disjointed sets: if a table matches two partition strategies,
	// an error will be raised at runtime.
	Tables []string `json:"tables,omitempty"`

	// Table glob patterns that should be skipped for this partitioning.
	//
	// If unset, no tables will be skipped.
	//
	// If a table matches both a pattern in `tables` and `skip_tables`, the table will be skipped.
	//
	// Partition strategy table patterns should be disjointed sets: if a table matches two partition strategies,
	// an error will be raised at runtime.
	SkipTables []string `json:"skip_tables,omitempty"`

	// Partitioning strategy to use, e.g. `toYYYYMM(_cq_sync_time)`,
	// the string is passed as is after "PARTITION BY" clause with no validation or quoting.
	//
	// An unset partition_by is not valid.
	PartitionBy string `json:"partition_by"`
}

type Spec

type Spec struct {
	// Connection string to connect to the database.
	// See [SDK documentation](https://github.com/ClickHouse/clickhouse-go#dsn) for more details.
	//
	// Example connection string:
	//
	// - `"clickhouse://username:password@host1:9000,host2:9000/database?dial_timeout=200ms&max_execution_time=60"`
	ConnectionString string `json:"connection_string,omitempty" jsonschema:"required,minLength=1"`

	// Cluster name to be used for [distributed DDL](https://clickhouse.com/docs/en/sql-reference/distributed-ddl).
	// If the value is empty, DDL operations will affect only the server the plugin is connected to.
	Cluster string `json:"cluster,omitempty"`

	// Engine to be used for tables.
	// Only [`*MergeTree` family](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family) is supported at the moment.
	Engine *Engine `json:"engine,omitempty"`

	// PEM-encoded certificate authorities.
	// When set, a certificate pool will be created by appending the certificates to the system pool.
	//
	// See [file variable substitution](/docs/advanced-topics/environment-variable-substitution#file-variable-substitution-example)
	// for how to read this value from a file.
	CACert string `json:"ca_cert,omitempty"`

	// Maximum number of items that may be grouped together to be written in a single write.
	BatchSize int64 `json:"batch_size,omitempty" jsonschema:"minimum=1,default=10000"`

	// Maximum size of items that may be grouped together to be written in a single write.
	BatchSizeBytes int64 `json:"batch_size_bytes,omitempty" jsonschema:"minimum=1,default=5242880"`

	// Maximum interval between batch writes.
	BatchTimeout *configtype.Duration `json:"batch_timeout,omitempty"`

	// Enables partitioning of tables via the `PARTITION BY` clause.
	Partition []PartitionStrategy `json:"partition,omitempty"`

	// Enables setting table sort keys via the `ORDER BY` clause.
	OrderBy []OrderByStrategy `json:"order,omitempty"`
}

CloudQuery ClickHouse destination plugin spec.

func (Spec) JSONSchemaExtend

func (Spec) JSONSchemaExtend(sc *jsonschema.Schema)

we need to set default for batch_timeout

func (*Spec) Options

func (s *Spec) Options() (*clickhouse.Options, error)

func (*Spec) SetDefaults

func (s *Spec) SetDefaults()

func (*Spec) Validate

func (s *Spec) Validate() error