datamodel

Documentation

Index

• Variables
• func InitJSONSchema()
• func InitTaskAirbyteCatalog()
• func ValidateJSONSchema(schema *jsonschema.Schema, msg proto.Message, emitUnpopulated bool) error
• func ValidateJSONSchemaString(schema string, data string) error
• func ValidateTaskAirbyteCatalog(task modelPB.ModelInstance_Task, batch *structpb.Value) error
• type AirbyteCatalog
• type AirbyteMessage
• type AirbyteRecordMessage
• type AirbyteStream
• type BaseDynamic
  • func (base *BaseDynamic) BeforeCreate(db *gorm.DB) error
• type BaseStatic
• type ConfiguredAirbyteCatalog
• type ConfiguredAirbyteStream
• type ConnectionType
  • func (c *ConnectionType) Scan(value interface{}) error
  • func (c ConnectionType) Value() (driver.Value, error)
• type Connector
• type ConnectorDefinition
• type ConnectorState
  • func (r *ConnectorState) Scan(value interface{}) error
  • func (r ConnectorState) Value() (driver.Value, error)
• type ConnectorType
  • func (c *ConnectorType) Scan(value interface{}) error
  • func (c ConnectorType) Value() (driver.Value, error)
• type ReleaseStage
  • func (r *ReleaseStage) Scan(value interface{}) error
  • func (r ReleaseStage) Value() (driver.Value, error)

Variables

var DstConnDefJSONSchema *jsonschema.Schema

DstConnDefJSONSchema represents the DestinationConnectorDefinition JSON Schema

var DstConnJSONSchema *jsonschema.Schema

DstConnJSONSchema represents the DestinationConnector JSON Schema

var SrcConnDefJSONSchema *jsonschema.Schema

SrcConnDefJSONSchema represents the SourceConnectorDefinition JSON Schema

var SrcConnJSONSchema *jsonschema.Schema

SrcConnJSONSchema represents the SourceConnector JSON Schema

var TaskAirbyteCatalog map[string]*AirbyteCatalog

TaskAirbyteCatalog stores the pre-defined task AirbyteCatalog

Functions

func InitJSONSchema

func InitJSONSchema()

InitJSONSchema initialise JSON Schema instances with the given files

func InitTaskAirbyteCatalog

func InitTaskAirbyteCatalog()

InitTaskAirbyteCatalog reads all task AirbyteCatalog files and stores the JSON content in the global TaskAirbyteCatalog variable

func ValidateJSONSchema

func ValidateJSONSchema(schema *jsonschema.Schema, msg proto.Message, emitUnpopulated bool) error

ValidateJSONSchema validates the Protobuf message data

func ValidateJSONSchemaString

func ValidateJSONSchemaString(schema string, data string) error

ValidateJSONSchemaString validates the string data given a string schema

func ValidateTaskAirbyteCatalog

func ValidateTaskAirbyteCatalog(task modelPB.ModelInstance_Task, batch *structpb.Value) error

ValidateTaskAirbyteCatalog validates the TaskAirbyteCatalog's JSON schema given the task type and the batch data (i.e., the output from model-backend trigger)

Types

type AirbyteCatalog

type AirbyteCatalog struct {
	Streams []AirbyteStream `json:"streams"`
}

AirbyteCatalog defines the AirbyteCatalog protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L212-L222

type AirbyteMessage

type AirbyteMessage struct {
	Type   string                `json:"type"`
	Record *AirbyteRecordMessage `json:"record"`
}

AirbyteMessage defines the AirbyteMessage protocol as in https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L13-L49

type AirbyteRecordMessage

type AirbyteRecordMessage struct {
	Stream    string          `json:"stream"`
	Data      json.RawMessage `json:"data"`
	EmittedAt int64           `json:"emitted_at"`
	Namespace string          `json:"namespace"`
}

AirbyteRecordMessage defines the RECORD type of AirbyteMessage, AirbyteRecordMessage, protocol as in https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L50-L70

type AirbyteStream

type AirbyteStream struct {
	Name                    string          `json:"name"`
	JSONSchema              json.RawMessage `json:"json_schema"`
	SupportedSyncModes      []string        `json:"supported_sync_modes"`
	SourceDefinedCursor     bool            `json:"source_defined_cursor"`
	DefaultCursorField      []string        `json:"default_cursor_field"`
	SourceDefinedPrimaryKey [][]string      `json:"source_defined_primary_key"`
	Namespace               string          `json:"namespace"`
}

AirbyteStream defines the AirbyteStream protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L223-L260

type BaseDynamic

type BaseDynamic struct {
	UID        uuid.UUID      `gorm:"type:uuid;primary_key;<-:create"` // allow read and create
	CreateTime time.Time      `gorm:"autoCreateTime:nano"`
	UpdateTime time.Time      `gorm:"autoUpdateTime:nano"`
	DeleteTime gorm.DeletedAt `sql:"index"`
}

BaseDynamic contains common columns for all tables with dynamic UUID as primary key generated when creating

func (*BaseDynamic) BeforeCreate

func (base *BaseDynamic) BeforeCreate(db *gorm.DB) error

BeforeCreate will set a UUID rather than numeric ID.

type BaseStatic

type BaseStatic struct {
	UID        uuid.UUID      `gorm:"type:uuid;primary_key;<-:create"` // allow read and create
	CreateTime time.Time      `gorm:"autoCreateTime:nano"`
	UpdateTime time.Time      `gorm:"autoUpdateTime:nano"`
	DeleteTime gorm.DeletedAt `sql:"index"`
}

BaseStatic contains common columns for all tables with static UUID as primary key

type ConfiguredAirbyteCatalog

type ConfiguredAirbyteCatalog struct {
	Streams []ConfiguredAirbyteStream `json:"streams"`
}

ConfiguredAirbyteCatalog defines the ConfiguredAirbyteCatalog protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L261-L271

type ConfiguredAirbyteStream

type ConfiguredAirbyteStream struct {
	Stream              *AirbyteStream `json:"stream"`
	SyncMode            string         `json:"sync_mode"`
	CursorField         []string       `json:"cursor_field"`
	DestinationSyncMode string         `json:"destination_sync_mode"`
	PrimaryKey          []string       `json:"primary_key"`
}

ConfiguredAirbyteStream defines the ConfiguredAirbyteStream protocol as in: https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/protocol-models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L272-L299

type ConnectionType

type ConnectionType connectorPB.ConnectionType

ConnectionType is an alias type for Protobuf enum ConnectionType

func (*ConnectionType) Scan

func (c *ConnectionType) Scan(value interface{}) error

Scan function for custom GORM type ConnectionType

func (ConnectionType) Value

func (c ConnectionType) Value() (driver.Value, error)

Value function for custom GORM type ConnectionType

type Connector

type Connector struct {
	BaseDynamic
	ID                     string
	Owner                  string
	ConnectorDefinitionUID uuid.UUID
	Description            sql.NullString
	Tombstone              bool
	Configuration          datatypes.JSON `gorm:"type:jsonb"`
	ConnectorType          ConnectorType  `sql:"type:valid_connector_type"`
	State                  ConnectorState `sql:"type:valid_state_type"`
}

Connector is the data model of the connector table

type ConnectorDefinition

type ConnectorDefinition struct {
	BaseStatic
	ID                   string
	Title                string
	DockerRepository     string
	DockerImageTag       string
	DocumentationURL     string
	Icon                 string
	Tombstone            bool
	Public               bool
	Custom               bool
	ReleaseDate          *time.Time
	Spec                 datatypes.JSON `gorm:"type:jsonb"`
	ResourceRequirements datatypes.JSON `gorm:"type:jsonb"`
	ConnectorType        ConnectorType  `sql:"type:valid_connector_type"`
	ConnectionType       ConnectionType `sql:"type:valid_connection_type"`
	ReleaseStage         ReleaseStage   `sql:"type:valid_release_stage"`
}

ConnectorDefinition is the data model of the connector_definition table

type ConnectorState

type ConnectorState connectorPB.Connector_State

ConnectorState is an alias type for Protobuf enum ConnectorState

func (*ConnectorState) Scan

func (r *ConnectorState) Scan(value interface{}) error

Scan function for custom GORM type ConnectorState

func (ConnectorState) Value

func (r ConnectorState) Value() (driver.Value, error)

Value function for custom GORM type ConnectorState

type ConnectorType

type ConnectorType connectorPB.ConnectorType

ConnectorType is an alias type for Protobuf enum ConnectorType

func (*ConnectorType) Scan

func (c *ConnectorType) Scan(value interface{}) error

Scan function for custom GORM type ConnectorType

func (ConnectorType) Value

func (c ConnectorType) Value() (driver.Value, error)

Value function for custom GORM type ConnectorType

type ReleaseStage

type ReleaseStage connectorPB.ReleaseStage

ReleaseStage is an alias type for Protobuf enum ReleaseStage

func (*ReleaseStage) Scan

func (r *ReleaseStage) Scan(value interface{}) error

Scan function for custom GORM type ReleaseStage

func (ReleaseStage) Value

func (r ReleaseStage) Value() (driver.Value, error)

Value function for custom GORM type ReleaseStage