airbyte

Documentation

Index

• func Init(logger *zap.Logger, options ConnectorOptions) base.IConnector
• func InitAirbyteCatalog(logger *zap.Logger)
• type AirbyteCatalog
• type AirbyteMessage
• type AirbyteRecordMessage
• type AirbyteStream
• type ConfiguredAirbyteCatalog
• type ConfiguredAirbyteStream
• type Connector
  • func (c *Connector) CreateExecution(defUID uuid.UUID, task string, config *structpb.Struct, logger *zap.Logger) (base.IExecution, error)
  • func (c *Connector) Test(defUID uuid.UUID, config *structpb.Struct, logger *zap.Logger) (pipelinePB.Connector_State, error)
• type ConnectorOptions
• type Execution
  • func (e *Execution) Execute(inputs []*structpb.Struct) ([]*structpb.Struct, error)

Functions

func Init

func Init(logger *zap.Logger, options ConnectorOptions) base.IConnector

func InitAirbyteCatalog

func InitAirbyteCatalog(logger *zap.Logger)

InitAirbyteCatalog reads all task AirbyteCatalog files and stores the JSON content in the global TaskAirbyteCatalog variable

Types

type AirbyteCatalog

type AirbyteCatalog struct {
	Streams []AirbyteStream `json:"streams"`
}

AirbyteCatalog defines the AirbyteCatalog protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L212-L222

var TaskOutputAirbyteCatalog AirbyteCatalog

TaskOutputAirbyteCatalog stores the pre-defined task AirbyteCatalog

type AirbyteMessage

type AirbyteMessage struct {
	Type   string                `json:"type"`
	Record *AirbyteRecordMessage `json:"record"`
}

AirbyteMessage defines the AirbyteMessage protocol as in https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L13-L49

type AirbyteRecordMessage

type AirbyteRecordMessage struct {
	Stream    string          `json:"stream"`
	Data      json.RawMessage `json:"data"`
	EmittedAt int64           `json:"emitted_at"`
}

AirbyteRecordMessage defines the RECORD type of AirbyteMessage, AirbyteRecordMessage, protocol as in (without namespace field) https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L50-L70

type AirbyteStream

type AirbyteStream struct {
	Name                    string          `json:"name"`
	JSONSchema              json.RawMessage `json:"json_schema"`
	SupportedSyncModes      []string        `json:"supported_sync_modes"`
	SourceDefinedCursor     bool            `json:"source_defined_cursor"`
	DefaultCursorField      []string        `json:"default_cursor_field"`
	SourceDefinedPrimaryKey [][]string      `json:"source_defined_primary_key"`
}

AirbyteStream defines the AirbyteStream protocol as in (without namespace field): https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L223-L260

type ConfiguredAirbyteCatalog

type ConfiguredAirbyteCatalog struct {
	Streams []ConfiguredAirbyteStream `json:"streams"`
}

ConfiguredAirbyteCatalog defines the ConfiguredAirbyteCatalog protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L261-L271

type ConfiguredAirbyteStream

type ConfiguredAirbyteStream struct {
	Stream              *AirbyteStream `json:"stream"`
	SyncMode            string         `json:"sync_mode"`
	CursorField         []string       `json:"cursor_field"`
	DestinationSyncMode string         `json:"destination_sync_mode"`
	PrimaryKey          []string       `json:"primary_key"`
}

ConfiguredAirbyteStream defines the ConfiguredAirbyteStream protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L272-L299

type Connector

type Connector struct {
	base.Connector
	// contains filtered or unexported fields
}

func (*Connector) CreateExecution

func (c *Connector) CreateExecution(defUID uuid.UUID, task string, config *structpb.Struct, logger *zap.Logger) (base.IExecution, error)

func (*Connector) Test

func (c *Connector) Test(defUID uuid.UUID, config *structpb.Struct, logger *zap.Logger) (pipelinePB.Connector_State, error)

type ConnectorOptions

type ConnectorOptions struct {
	MountSourceVDP        string
	MountTargetVDP        string
	MountSourceAirbyte    string
	MountTargetAirbyte    string
	ExcludeLocalConnector bool
}

type Execution

type Execution struct {
	base.Execution
	// contains filtered or unexported fields
}

func (*Execution) Execute

func (e *Execution) Execute(inputs []*structpb.Struct) ([]*structpb.Struct, error)