pipeline

Oct 22, 2024

Changes in this version

+ const AssetTypeDuckDBQuery

Oct 3, 2024

Changes in this version

type SecretMapping

+ func (s SecretMapping) MarshalYAML() (interface{}, error)

Sep 25, 2024

Changes in this version

+ type LineageExtractor struct

+ type Parser interface

+ ColumnLineage func(sql, dialect string, schema sqlparser.Schema) (*sqlparser.Lineage, error)

Sep 21, 2024

Changes in this version

+ type DiscordNotification struct

+ Connection string

+ type MetadataPush struct

+ BigQuery bool

+ func (mp *MetadataPush) HasAnyEnabled() bool

type Notifications

+ Discord []DiscordNotification

type Pipeline

+ MetadataPush MetadataPush

Sep 17, 2024

Changes in this version

+ var SupportedFileSuffixes = []string

+ func ClearSpacesAtLineEndings(content string) string

type Asset

+ func (a *Asset) Persist(fs afero.Fs) error

type Column

+ Extends string

type ColumnCheckValue

+ func (ccv ColumnCheckValue) MarshalYAML() (interface{}, error)

type DefaultTrueBool

+ func (b *DefaultTrueBool) Bool() bool

+ func (b DefaultTrueBool) MarshalYAML() (interface{}, error)

type Upstream

+ func (u Upstream) MarshalYAML() (interface{}, error)

Sep 2, 2024

Changes in this version

+ type DefaultTrueBool struct

+ Value *bool

+ func (b *DefaultTrueBool) UnmarshalJSON(data []byte) error

+ func (b *DefaultTrueBool) UnmarshalYAML(value *yaml.Node) error

+ func (b DefaultTrueBool) MarshalJSON() ([]byte, error)

+ type NotificationCommon struct

+ Failure DefaultTrueBool

+ Success DefaultTrueBool

Aug 23, 2024

Changes in this version

+ const AssetTypeAthenaQuery

+ const AssetTypeDatabricksQuery

type Asset

+ Athena AthenaConfig

+ type AthenaConfig struct

+ Location string

+ func (s AthenaConfig) MarshalJSON() ([]byte, error)

+ type MSTeamsNotification struct

+ Connection string

type Notifications

+ MSTeams []MSTeamsNotification

Jul 25, 2024

Changes in this version

type Pipeline

+ func (p *Pipeline) WipeContentOfAssets()

Jul 18, 2024

Changes in this version

type Asset

+ Snowflake SnowflakeConfig

+ type SnowflakeConfig struct

+ Warehouse string

+ func (s SnowflakeConfig) MarshalJSON() ([]byte, error)

Jul 11, 2024

Changes in this version

type Asset

+ URI string

+ func (a *Asset) GetColumnWithName(name string) *Column

type Upstream

+ Metadata EmptyStringMap

Jul 2, 2024

Changes in this version

type Asset

+ Upstreams []Upstream

+ func (a *Asset) EnrichFromEntityAttributes(entities []*glossary.Entity) error

+ func (a Asset) MarshalJSON() ([]byte, error)

+ type Builder struct

+ GlossaryReader glossaryReader

+ func (b *Builder) CreateAssetFromFile(path string) (*Asset, error)

+ func (b *Builder) CreatePipelineFromPath(pathToPipeline string) (*Pipeline, error)

+ func (b *Builder) SetGlossaryReader(reader glossaryReader)

type Column

+ EntityAttribute *EntityAttribute

+ type EntityAttribute struct

+ Attribute string

+ Entity string

+ type Upstream struct

+ Type string

+ Value string

Jun 24, 2024

Changes in this version

+ const RunConfigEndDate

+ const RunConfigStartDate

Jun 24, 2024

Changes in this version

type Pipeline

+ func PipelineFromPath(filePath string, fs afero.Fs) (*Pipeline, error)

Jun 4, 2024

Changes in this version

type Asset

+ Tags EmptyStringArray

+ type EmptyStringArray []string

+ func (a *EmptyStringArray) UnmarshalJSON(data []byte) error

+ func (a EmptyStringArray) MarshalJSON() ([]byte, error)

type Pipeline

+ func (p *Pipeline) GetAssetsByTag(tag string) []*Asset

May 25, 2024

Changes in this version

type CustomCheck

+ Description string

May 22, 2024

Changes in this version

+ const RunConfigFullRefresh

+ type RunConfig string

May 15, 2024

Changes in this version

type ColumnCheckValue

+ func (ccv *ColumnCheckValue) ToString() string

+ type ParseError struct

+ Msg string

+ func (e *ParseError) Error() string

May 9, 2024

Changes in this version

type ColumnCheck

+ Blocking bool

type CustomCheck

+ Blocking bool

May 6, 2024

Changes in this version

+ const AssetTypeBigqueryQuerySensor

May 2, 2024

Changes in this version

type Pipeline

+ func (p *Pipeline) GetMajorityAssetTypesFromSQLAssets(defaultIfNone AssetType) AssetType

May 2, 2024

Changes in this version

type Column

+ func (c *Column) HasCheck(check string) bool

Apr 15, 2024

Changes in this version

+ const AssetTypeBigqueryTableSensor

Apr 8, 2024

Changes in this version

+ var IngestrTypeConnectionMapping = map[string]AssetType

Mar 22, 2024

Changes in this version

type Asset

+ Instance string

+ Owner string

Mar 20, 2024

Changes in this version

+ var AssetTypeConnectionMapping = map[AssetType][]string

Mar 14, 2024

Changes in this version

+ const AssetTypeIngestr

+ type AssetCollection []*Asset

+ func (ac AssetCollection) MarshalJSON() ([]byte, error)

type Materializer

+ FullRefresh bool

Feb 23, 2024

Changes in this version

+ const AssetTypeMsSQLQuery

+ const AssetTypeSynapseQuery

type Pipeline

+ func (p *Pipeline) GetAssetByName(assetName string) *Asset

Feb 7, 2024

Changes in this version

+ const AssetTypePostgresQuery

+ const AssetTypeRedshiftQuery

+ var AllAvailableMaterializationStrategies = []MaterializationStrategy

type Asset

+ func (a *Asset) ColumnNames() []string

+ func (a *Asset) ColumnNamesWithPrimaryKey() []string

+ func (a *Asset) ColumnNamesWithUpdateOnMerge() []string

+ type AssetMaterializationMap map[MaterializationType]map[MaterializationStrategy]MaterializerFunc

type Column

+ UpdateOnMerge bool

type ColumnCheckValue

+ func (ccv *ColumnCheckValue) UnmarshalJSON(data []byte) error

type EmptyStringMap

+ func (b *EmptyStringMap) UnmarshalJSON(data []byte) error

type MaterializationStrategy

+ const MaterializationStrategyMerge

+ type Materializer struct

+ MaterializationMap AssetMaterializationMap

+ func (m *Materializer) Render(asset *Asset, query string) (string, error)

+ type MaterializerFunc func(task *Asset, query string) (string, error)

Feb 1, 2024

Changes in this version

type Column

+ PrimaryKey bool

Jan 31, 2024

Changes in this version

+ const AssetTypeSnowflakeQuerySensor

Jan 22, 2024

Changes in this version

type Pipeline

+ Catchup bool

+ Retries int

Jan 17, 2024

Changes in this version

type Column

+ Type string

type ColumnCheck

+ func NewColumnCheck(assetName, columnName, name string, value ColumnCheckValue) ColumnCheck

Dec 22, 2023

Changes in this version

type Asset

+ ID string

Dec 8, 2023

Changes in this version

+ const AssetTypeBigqueryQuery

+ const AssetTypeEmpty

+ const AssetTypePython

+ const AssetTypeSnowflakeQuery

+ const CommentTask

+ const YamlTask

+ var ValidQualityChecks = map[string]bool

+ func NewBuilder(config BuilderConfig, yamlTaskCreator TaskCreator, ...) *builder

+ type Asset struct

+ Columns []Column

+ Connection string

+ CustomChecks []CustomCheck

+ DefinitionFile TaskDefinitionFile

+ DependsOn []string

+ Description string

+ ExecutableFile ExecutableFile

+ Image string

+ Materialization Materialization

+ Metadata EmptyStringMap

+ Name string

+ Parameters map[string]string

+ Pipeline *Pipeline

+ Secrets []SecretMapping

+ Type AssetType

+ func ConvertYamlToTask(content []byte) (*Asset, error)

+ func (a *Asset) AddDownstream(asset *Asset)

+ func (a *Asset) AddUpstream(asset *Asset)

+ func (a *Asset) GetDownstream() []*Asset

+ func (a *Asset) GetFullDownstream() []*Asset

+ func (a *Asset) GetFullUpstream() []*Asset

+ func (a *Asset) GetUpstream() []*Asset

+ type AssetType string

+ type BuilderConfig struct

+ PipelineFileName string

+ TasksDirectoryName string

+ TasksDirectoryNames []string

+ TasksFileSuffixes []string

+ type Column struct

+ Checks []ColumnCheck

+ Description string

+ Name string

+ type ColumnCheck struct

+ ID string

+ Name string

+ Value ColumnCheckValue

+ type ColumnCheckValue struct

+ Bool *bool

+ Float *float64

+ Int *int

+ IntArray *[]int

+ String *string

+ StringArray *[]string

+ func (v *ColumnCheckValue) MarshalJSON() ([]byte, error)

+ type CustomCheck struct

+ ID string

+ Name string

+ Query string

+ Value ColumnCheckValue

+ type DefinitionFile struct

+ Name string

+ Path string

+ type EmptyStringMap map[string]string

+ func (m EmptyStringMap) MarshalJSON() ([]byte, error)

+ type ExecutableFile struct

+ Content string

+ Name string

+ Path string

+ type Materialization struct

+ ClusterBy []string

+ IncrementalKey string

+ PartitionBy string

+ Strategy MaterializationStrategy

+ Type MaterializationType

+ func (m Materialization) MarshalJSON() ([]byte, error)

+ type MaterializationStrategy string

+ const MaterializationStrategyAppend

+ const MaterializationStrategyCreateReplace

+ const MaterializationStrategyDeleteInsert

+ const MaterializationStrategyNone

+ type MaterializationType string

+ const MaterializationTypeNone

+ const MaterializationTypeTable

+ const MaterializationTypeView

+ type Notifications struct

+ Slack []SlackNotification

+ func (n Notifications) MarshalJSON() ([]byte, error)

+ type Pipeline struct

+ Assets []*Asset

+ DefaultConnections EmptyStringMap

+ DefaultParameters EmptyStringMap

+ DefinitionFile DefinitionFile

+ LegacyID string

+ Name string

+ Notifications Notifications

+ Schedule schedule

+ StartDate string

+ TasksByType map[AssetType][]*Asset

+ func (p *Pipeline) GetAssetByPath(assetPath string) *Asset

+ func (p *Pipeline) GetConnectionNameForAsset(asset *Asset) string

+ func (p *Pipeline) HasAssetType(taskType AssetType) bool

+ func (p *Pipeline) RelativeAssetPath(t *Asset) string

+ type SecretMapping struct

+ InjectedKey string

+ SecretKey string

+ type SlackNotification struct

+ Channel string

+ type TaskCreator func(path string) (*Asset, error)

+ func CreateTaskFromFileComments(fs afero.Fs) TaskCreator

+ func CreateTaskFromYamlDefinition(fs afero.Fs) TaskCreator

+ type TaskDefinitionFile struct

+ Name string

+ Path string

+ Type TaskDefinitionType

+ type TaskDefinitionType string

+ type TaskSchedule struct

+ Days []string