airbyte

Documentation

Index

• Constants
• func ClientSend(url string, key string, cnf interface{}, in io.Reader) error
• func Destination(w io.Writer) error
• func DestinationCheck(w io.Writer, cnf DestinationConfig) error
• func DestinationSpec(w io.Writer) error
• func DestinationWrite(r io.Reader, cnf WriteConfig) error
• func MapToStrings(N int, mapper func(i int) string) []string
• func ReadJson(pathOrConfig string, v interface{}) error
• func Source(w io.Writer) error
• func SourceCheck(w io.Writer, cnf SourceConfig) error
• func SourceDiscover(w io.Writer, cnf SourceConfig) error
• func SourceRead(w io.Writer, cnf ReadConfig) error
• func SourceSpec(w io.Writer) error
• func ToStrings(strs [][]byte) []string
• type AirbyteCatalog
• type AirbyteCatalogWrapper
• type AirbyteLogMessage
• type AirbyteLogMessageWrap
• type AirbyteStream
• type ArrProperty
• type ConfiguredAirbyteCatalog
• type ConfiguredAirbyteStream
• type ConnectionSpecification
• type ConnectionStatus
• type ConnectionStatusWrap
• type ConnectorSpecification
• type ConnectorSpecificationWrap
• type DestinationConfig
• type JSONSchema
• type Property
• type ReadConfig
• type SourceConfig
• type SourceState
• type SourceStateData
• type SourceStateWrap
• type WriteConfig

Constants

const UniqErrStr = "APPERROR" // simplify search in logs.. :)

Functions

func ClientSend

func ClientSend(url string, key string, cnf interface{}, in io.Reader) error

curl -F 'file_field=@../../../platform-data/rusta/se_trans2.csv' http://127.0.0.1:8082

func Destination

func Destination(w io.Writer) error

func DestinationCheck

func DestinationCheck(w io.Writer, cnf DestinationConfig) error

func DestinationSpec

func DestinationSpec(w io.Writer) error

func DestinationWrite

func DestinationWrite(r io.Reader, cnf WriteConfig) error

func MapToStrings

func MapToStrings(N int, mapper func(i int) string) []string

func ReadJson

func ReadJson(pathOrConfig string, v interface{}) error

func Source

func Source(w io.Writer) error

func SourceCheck

func SourceCheck(w io.Writer, cnf SourceConfig) error

func SourceDiscover

func SourceDiscover(w io.Writer, cnf SourceConfig) error

func SourceRead

func SourceRead(w io.Writer, cnf ReadConfig) error

func SourceSpec

func SourceSpec(w io.Writer) error

func ToStrings

func ToStrings(strs [][]byte) []string

Types

type AirbyteCatalog

type AirbyteCatalog struct {
	Streams []AirbyteStream `json:"streams"`
}

type AirbyteCatalogWrapper

type AirbyteCatalogWrapper struct {
	Type    string         `json:"type"`
	Catalog AirbyteCatalog `json:"catalog"`
}

type AirbyteLogMessage

type AirbyteLogMessage struct {
	Level   string `json:"level"`
	Message string `json:"message"`
}

type AirbyteLogMessageWrap

type AirbyteLogMessageWrap struct {
	Type string            `json:"type"`
	Log  AirbyteLogMessage `json:"log"`
}

type AirbyteStream

type AirbyteStream struct {
	Name                    string     `json:"name"`
	Namespace               string     `json:"namespace"`
	Schema                  JSONSchema `json:"json_schema"`
	SupportedSyncModes      []string   `json:"supported_sync_modes,omitempty"` //["full_refresh", "incremental"]
	SourceDefinedCursor     bool       `json:"source_defined_cursor,omitempty"`
	DefaultCursorField      []string   `json:"default_cursor_field,omitempty"`
	SourceDefinedPrimaryKey [][]string `json:"source_defined_primary_key,omitempty"` //[["resultId"], ["resultIdx"], ["i64"]]
}

type ArrProperty

type ArrProperty struct {
	Type       interface{}         `json:"type"` // string or []string
	Properties map[string]Property `json:"properties"`
}

type ConfiguredAirbyteCatalog

type ConfiguredAirbyteCatalog struct {
	Streams []ConfiguredAirbyteStream `json:"streams"`
}

type ConfiguredAirbyteStream

type ConfiguredAirbyteStream struct {
	Stream              AirbyteStream `json:"stream"`
	SyncMode            string        `json:"sync_mode"`             //full_refresh [full_refresh,incremental]
	CursorField         []string      `json:"cursor_field"`          //Path to the field that will be used to determine if a record is new or modified since the last sync. This field is REQUIRED if `sync_mode` is `incremental`. Otherwise it is ignored.
	DestinationSyncMode string        `json:"destination_sync_mode"` //append [append, overwrite, append_dedup, upsert_dedup]
	PrimaryKey          [][]string    `json:"primary_key"`           // Paths to the fields that will be used as primary key. This field is REQUIRED if `destination_sync_mode` is `*_dedup`. Otherwise it is ignored.
}

required: stream, sync_mode, destination_sync_mode

type ConnectionSpecification

type ConnectionSpecification struct {
	Schema               string              `json:"$schema"`
	Title                string              `json:"title"`
	Type                 string              `json:"type"`
	Required             []string            `json:"required"`
	AdditionalProperties bool                `json:"additionalProperties"`
	Properties           map[string]Property `json:"properties"`
}

type ConnectionStatus

type ConnectionStatus struct {
	Status  string `json:"status"`
	Message string `json:"message,omitempty"`
}

type ConnectionStatusWrap

type ConnectionStatusWrap struct {
	Type             string           `json:"type"`
	ConnectionStatus ConnectionStatus `json:"connectionStatus"`
}

type ConnectorSpecification

type ConnectorSpecification struct {
	DocumentationUrl              string                  `json:"documentationUrl"`
	SupportsIncremental           bool                    `json:"supportsIncremental,omitempty"`
	SupportedDestinationSyncModes []string                `json:"supported_destination_sync_modes,omitempty"`
	ConnectionSpecification       ConnectionSpecification `json:"connectionSpecification"`
}

type ConnectorSpecificationWrap

type ConnectorSpecificationWrap struct {
	Type string                 `json:"type"`
	Spec ConnectorSpecification `json:"spec"`
}

type DestinationConfig

type DestinationConfig struct {
	Url string `json:"url"`
	Key string `json:"key"`
}

type JSONSchema

type JSONSchema struct {
	Schema     string              `json:"$schema"`
	Type       string              `json:"type"`
	Properties map[string]Property `json:"properties"`
}

type Property

type Property struct {
	Description string              `json:"description,omitempty"`
	Type        interface{}         `json:"type"` // string or []string
	Examples    []string            `json:"examples,omitempty"`
	Properties  map[string]Property `json:"properties"`
	Items       ArrProperty         `json:"items"`
}

type ReadConfig

type ReadConfig struct {
	Config  SourceConfig             `json:"config"`
	Catalog ConfiguredAirbyteCatalog `json:"catalog"`
	State   interface{}              `json:"state"`
}

type SourceConfig

type SourceConfig struct {
	Url string `json:"url"`
	Key string `json:"key"`
}

type SourceState

type SourceState struct {
	Hello string
	Info  string
}

type SourceStateData

type SourceStateData struct {
	Data SourceState `json:"data"`
}

type SourceStateWrap

type SourceStateWrap struct {
	Type  string          `json:"type"`
	State SourceStateData `json:"state"`
}

type WriteConfig

type WriteConfig struct {
	Config  DestinationConfig        `json:"config"`
	Catalog ConfiguredAirbyteCatalog `json:"catalog"`
}