datamodel

Documentation

Index

• Variables
• func InitAirbyteCatalog()
• func InitJSONSchema()
• func ValidateAirbyteCatalog(taskOutputs []*pipelinePB.TaskOutput) error
• func ValidateJSONSchema(schema *jsonschema.Schema, msg proto.Message, emitUnpopulated bool) error
• func ValidateJSONSchemaString(schema string, data *structpb.Struct) error
• type AirbyteCatalog
• type AirbyteMessage
• type AirbyteRecordMessage
• type AirbyteStream
• type BaseDynamic
  • func (base *BaseDynamic) BeforeCreate(db *gorm.DB) error
• type BaseStatic
• type ConfiguredAirbyteCatalog
• type ConfiguredAirbyteStream
• type Connector
• type ConnectorDefinition
• type ConnectorState
  • func (r *ConnectorState) Scan(value interface{}) error
  • func (r ConnectorState) Value() (driver.Value, error)
• type ConnectorType
  • func (c *ConnectorType) Scan(value interface{}) error
  • func (c ConnectorType) Value() (driver.Value, error)
• type ReleaseStage
  • func (r *ReleaseStage) Scan(value interface{}) error
  • func (r ReleaseStage) Value() (driver.Value, error)
• type WriteDestinationConnectorParam

Variables

var DstConnDefJSONSchema *jsonschema.Schema

DstConnDefJSONSchema represents the DestinationConnectorDefinition JSON Schema

var DstConnJSONSchema *jsonschema.Schema

DstConnJSONSchema represents the DestinationConnector JSON Schema

var SrcConnDefJSONSchema *jsonschema.Schema

SrcConnDefJSONSchema represents the SourceConnectorDefinition JSON Schema

var SrcConnJSONSchema *jsonschema.Schema

SrcConnJSONSchema represents the SourceConnector JSON Schema

Functions

func InitAirbyteCatalog

func InitAirbyteCatalog()

InitAirbyteCatalog reads all task AirbyteCatalog files and stores the JSON content in the global TaskAirbyteCatalog variable

func InitJSONSchema

func InitJSONSchema()

InitJSONSchema initialise JSON Schema instances with the given files

func ValidateAirbyteCatalog

func ValidateAirbyteCatalog(taskOutputs []*pipelinePB.TaskOutput) error

ValidateAirbyteCatalog validates the TaskAirbyteCatalog's JSON schema given the task type and the batch data (i.e., the output from model-backend trigger)

func ValidateJSONSchema

func ValidateJSONSchema(schema *jsonschema.Schema, msg proto.Message, emitUnpopulated bool) error

ValidateJSONSchema validates the Protobuf message data

func ValidateJSONSchemaString

func ValidateJSONSchemaString(schema string, data *structpb.Struct) error

ValidateJSONSchemaString validates the string data given a string schema

Types

type AirbyteCatalog

type AirbyteCatalog struct {
	Streams []AirbyteStream `json:"streams"`
}

AirbyteCatalog defines the AirbyteCatalog protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L212-L222

var TaskOutputAirbyteCatalog AirbyteCatalog

TaskOutputAirbyteCatalog stores the pre-defined task AirbyteCatalog

type AirbyteMessage

type AirbyteMessage struct {
	Type   string                `json:"type"`
	Record *AirbyteRecordMessage `json:"record"`
}

AirbyteMessage defines the AirbyteMessage protocol as in https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L13-L49

type AirbyteRecordMessage

type AirbyteRecordMessage struct {
	Stream    string          `json:"stream"`
	Data      json.RawMessage `json:"data"`
	EmittedAt int64           `json:"emitted_at"`
}

AirbyteRecordMessage defines the RECORD type of AirbyteMessage, AirbyteRecordMessage, protocol as in (without namespace field) https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L50-L70

type AirbyteStream

type AirbyteStream struct {
	Name                    string          `json:"name"`
	JSONSchema              json.RawMessage `json:"json_schema"`
	SupportedSyncModes      []string        `json:"supported_sync_modes"`
	SourceDefinedCursor     bool            `json:"source_defined_cursor"`
	DefaultCursorField      []string        `json:"default_cursor_field"`
	SourceDefinedPrimaryKey [][]string      `json:"source_defined_primary_key"`
}

AirbyteStream defines the AirbyteStream protocol as in (without namespace field): https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L223-L260

type BaseDynamic

type BaseDynamic struct {
	UID        uuid.UUID      `gorm:"type:uuid;primary_key;<-:create"` // allow read and create
	CreateTime time.Time      `gorm:"autoCreateTime:nano"`
	UpdateTime time.Time      `gorm:"autoUpdateTime:nano"`
	DeleteTime gorm.DeletedAt `sql:"index"`
}

BaseDynamic contains common columns for all tables with dynamic UUID as primary key generated when creating

func (*BaseDynamic) BeforeCreate

func (base *BaseDynamic) BeforeCreate(db *gorm.DB) error

BeforeCreate will set a UUID rather than numeric ID.

type BaseStatic

type BaseStatic struct {
	UID        uuid.UUID      `gorm:"type:uuid;primary_key;<-:create"` // allow read and create
	CreateTime time.Time      `gorm:"autoCreateTime:nano"`
	UpdateTime time.Time      `gorm:"autoUpdateTime:nano"`
	DeleteTime gorm.DeletedAt `sql:"index"`
}

BaseStatic contains common columns for all tables with static UUID as primary key

type ConfiguredAirbyteCatalog

type ConfiguredAirbyteCatalog struct {
	Streams []ConfiguredAirbyteStream `json:"streams"`
}

ConfiguredAirbyteCatalog defines the ConfiguredAirbyteCatalog protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L261-L271

type ConfiguredAirbyteStream

type ConfiguredAirbyteStream struct {
	Stream              *AirbyteStream `json:"stream"`
	SyncMode            string         `json:"sync_mode"`
	CursorField         []string       `json:"cursor_field"`
	DestinationSyncMode string         `json:"destination_sync_mode"`
	PrimaryKey          []string       `json:"primary_key"`
}

ConfiguredAirbyteStream defines the ConfiguredAirbyteStream protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L272-L299

type Connector

type Connector struct {
	BaseDynamic
	ID                     string
	Owner                  string
	ConnectorDefinitionUID uuid.UUID
	Description            sql.NullString
	Tombstone              bool
	Configuration          datatypes.JSON `gorm:"type:jsonb"`
	ConnectorType          ConnectorType  `sql:"type:valid_connector_type"`
	State                  ConnectorState `sql:"type:valid_state_type"`
}

Connector is the data model of the connector table

type ConnectorDefinition

type ConnectorDefinition struct {
	BaseStatic
	ID                   string
	Title                string
	DockerRepository     string
	DockerImageTag       string
	DocumentationURL     string
	Icon                 string
	Tombstone            bool
	Public               bool
	Custom               bool
	ReleaseDate          *time.Time
	Spec                 datatypes.JSON `gorm:"type:jsonb"`
	ResourceRequirements datatypes.JSON `gorm:"type:jsonb"`
	ConnectorType        ConnectorType  `sql:"type:valid_connector_type"`
	ReleaseStage         ReleaseStage   `sql:"type:valid_release_stage"`
}

ConnectorDefinition is the data model of the connector_definition table

type ConnectorState

type ConnectorState connectorPB.Connector_State

ConnectorState is an alias type for Protobuf enum ConnectorState

func (*ConnectorState) Scan

func (r *ConnectorState) Scan(value interface{}) error

Scan function for custom GORM type ConnectorState

func (ConnectorState) Value

func (r ConnectorState) Value() (driver.Value, error)

Value function for custom GORM type ConnectorState

type ConnectorType

type ConnectorType connectorPB.ConnectorType

ConnectorType is an alias type for Protobuf enum ConnectorType

func (*ConnectorType) Scan

func (c *ConnectorType) Scan(value interface{}) error

Scan function for custom GORM type ConnectorType

func (ConnectorType) Value

func (c ConnectorType) Value() (driver.Value, error)

Value function for custom GORM type ConnectorType

type ReleaseStage

type ReleaseStage connectorPB.ReleaseStage

ReleaseStage is an alias type for Protobuf enum ReleaseStage

func (*ReleaseStage) Scan

func (r *ReleaseStage) Scan(value interface{}) error

Scan function for custom GORM type ReleaseStage

func (ReleaseStage) Value

func (r ReleaseStage) Value() (driver.Value, error)

Value function for custom GORM type ReleaseStage

type WriteDestinationConnectorParam

type WriteDestinationConnectorParam struct {
	SyncMode             string
	DstSyncMode          string
	Pipeline             string
	Recipe               *pipelinePB.Recipe
	DataMappingIndices   []string
	ModelInstanceOutputs []*pipelinePB.ModelInstanceOutput
}

WriteDestinationConnectorParam stores the parameters for WriteDestinationConnector service per model instance