lint

Documentation

Index

• func EnsureNoNestedPipelines(pipelinePaths []string) error
• type Issue
  • func EnsureAthenaSQLTypeTasksHasDatabaseAndS3FilePath(p *pipeline.Pipeline) ([]*Issue, error)
  • func EnsureDependencyExists(p *pipeline.Pipeline) ([]*Issue, error)
  • func EnsureMaterializationValuesAreValid(p *pipeline.Pipeline) ([]*Issue, error)
  • func EnsureOnlyAcceptedTaskTypesAreThere(p *pipeline.Pipeline) ([]*Issue, error)
  • func EnsurePipelineHasNoCycles(p *pipeline.Pipeline) ([]*Issue, error)
  • func EnsurePipelineNameIsValid(pipeline *pipeline.Pipeline) ([]*Issue, error)
  • func EnsurePipelineScheduleIsValidCron(p *pipeline.Pipeline) ([]*Issue, error)
  • func EnsureSlackFieldInPipelineIsValid(p *pipeline.Pipeline) ([]*Issue, error)
  • func EnsureSnowflakeSensorHasQueryParameter(p *pipeline.Pipeline) ([]*Issue, error)
  • func EnsureTaskNameIsUnique(p *pipeline.Pipeline) ([]*Issue, error)
  • func EnsureTaskNameIsValid(pipeline *pipeline.Pipeline) ([]*Issue, error)
• type Linter
  • func NewLinter(findPipelines pipelineFinder, builder pipelineBuilder, rules []Rule, ...) *Linter
  • func (l *Linter) Lint(rootPath, pipelineDefinitionFileName string) (*PipelineAnalysisResult, error)
  • func (l *Linter) LintPipeline(p *pipeline.Pipeline) (*PipelineIssues, error)
  • func (l *Linter) LintPipelines(pipelines []*pipeline.Pipeline) (*PipelineAnalysisResult, error)
• type PipelineAnalysisResult
  • func (p *PipelineAnalysisResult) ErrorCount() int
• type PipelineIssues
  • func RunLintRulesOnPipeline(p *pipeline.Pipeline, rules []Rule) (*PipelineIssues, error)
• type PipelineValidator
  • func EnsureExecutableFileIsValid(fs afero.Fs) PipelineValidator
• type Printer
  • func (l *Printer) PrintIssues(analysis *PipelineAnalysisResult)
• type QueryValidatorRule
  • func (q *QueryValidatorRule) Name() string
  • func (q *QueryValidatorRule) Validate(p *pipeline.Pipeline) ([]*Issue, error)
• type Rule
  • func GetRules(fs afero.Fs) ([]Rule, error)
• type SimpleRule
  • func (g *SimpleRule) Name() string
  • func (g *SimpleRule) Validate(pipeline *pipeline.Pipeline) ([]*Issue, error)

Functions

func EnsureNoNestedPipelines

func EnsureNoNestedPipelines(pipelinePaths []string) error

Types

type Issue

type Issue struct {
	Task        *pipeline.Asset
	Description string
	Context     []string
}

func EnsureAthenaSQLTypeTasksHasDatabaseAndS3FilePath

func EnsureAthenaSQLTypeTasksHasDatabaseAndS3FilePath(p *pipeline.Pipeline) ([]*Issue, error)

func EnsureDependencyExists

func EnsureDependencyExists(p *pipeline.Pipeline) ([]*Issue, error)

func EnsureMaterializationValuesAreValid

func EnsureMaterializationValuesAreValid(p *pipeline.Pipeline) ([]*Issue, error)

func EnsureOnlyAcceptedTaskTypesAreThere

func EnsureOnlyAcceptedTaskTypesAreThere(p *pipeline.Pipeline) ([]*Issue, error)

func EnsurePipelineHasNoCycles

func EnsurePipelineHasNoCycles(p *pipeline.Pipeline) ([]*Issue, error)

EnsurePipelineHasNoCycles ensures that the pipeline is a DAG, and contains no cycles. Since the pipelines are directed graphs, strongly connected components mean cycles, therefore they would be considered invalid for our pipelines. Strong connectivity wouldn't work for tasks that depend on themselves, therefore there's a specific check for that.

func EnsurePipelineNameIsValid

func EnsurePipelineNameIsValid(pipeline *pipeline.Pipeline) ([]*Issue, error)

func EnsurePipelineScheduleIsValidCron

func EnsurePipelineScheduleIsValidCron(p *pipeline.Pipeline) ([]*Issue, error)

func EnsureSlackFieldInPipelineIsValid

func EnsureSlackFieldInPipelineIsValid(p *pipeline.Pipeline) ([]*Issue, error)

func EnsureSnowflakeSensorHasQueryParameter

func EnsureSnowflakeSensorHasQueryParameter(p *pipeline.Pipeline) ([]*Issue, error)

func EnsureTaskNameIsUnique

func EnsureTaskNameIsUnique(p *pipeline.Pipeline) ([]*Issue, error)

func EnsureTaskNameIsValid

func EnsureTaskNameIsValid(pipeline *pipeline.Pipeline) ([]*Issue, error)

type Linter

type Linter struct {
	// contains filtered or unexported fields
}

func NewLinter

func NewLinter(findPipelines pipelineFinder, builder pipelineBuilder, rules []Rule, logger *zap.SugaredLogger) *Linter

func (*Linter) Lint

func (l *Linter) Lint(rootPath, pipelineDefinitionFileName string) (*PipelineAnalysisResult, error)

func (*Linter) LintPipeline

func (l *Linter) LintPipeline(p *pipeline.Pipeline) (*PipelineIssues, error)

func (*Linter) LintPipelines

func (l *Linter) LintPipelines(pipelines []*pipeline.Pipeline) (*PipelineAnalysisResult, error)

type PipelineAnalysisResult

type PipelineAnalysisResult struct {
	Pipelines []*PipelineIssues
}

func (*PipelineAnalysisResult) ErrorCount

func (p *PipelineAnalysisResult) ErrorCount() int

ErrorCount returns the number of errors found in an analysis result.

type PipelineIssues

type PipelineIssues struct {
	Pipeline *pipeline.Pipeline
	Issues   map[Rule][]*Issue
}

func RunLintRulesOnPipeline

func RunLintRulesOnPipeline(p *pipeline.Pipeline, rules []Rule) (*PipelineIssues, error)

type PipelineValidator

type PipelineValidator func(pipeline *pipeline.Pipeline) ([]*Issue, error)

func EnsureExecutableFileIsValid

func EnsureExecutableFileIsValid(fs afero.Fs) PipelineValidator

type Printer

type Printer struct {
	RootCheckPath string
}

func (*Printer) PrintIssues

func (l *Printer) PrintIssues(analysis *PipelineAnalysisResult)

type QueryValidatorRule

type QueryValidatorRule struct {
	Identifier  string
	TaskType    pipeline.AssetType
	Connections connectionManager
	Extractor   queryExtractor
	WorkerCount int
	Logger      *zap.SugaredLogger
}

func (*QueryValidatorRule) Name

func (q *QueryValidatorRule) Name() string

func (*QueryValidatorRule) Validate

func (q *QueryValidatorRule) Validate(p *pipeline.Pipeline) ([]*Issue, error)

type Rule

type Rule interface {
	Name() string
	Validate(pipeline *pipeline.Pipeline) ([]*Issue, error)
}

func GetRules

func GetRules(fs afero.Fs) ([]Rule, error)

type SimpleRule

type SimpleRule struct {
	Identifier string
	Validator  PipelineValidator
}

func (*SimpleRule) Name

func (g *SimpleRule) Name() string

func (*SimpleRule) Validate

func (g *SimpleRule) Validate(pipeline *pipeline.Pipeline) ([]*Issue, error)