rillv1

package
v0.48.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 28, 2024 License: Apache-2.0 Imports: 38 Imported by: 0

Documentation

Index

Constants

View Source 
const Version = "rillv1"

Version identifier for this compiler

Variables

View Source 
var ErrRillYAMLNotFound = errors.New("rill.yaml not found")

Functions

func EvaluateBoolExpression added in v0.33.0 

func EvaluateBoolExpression(expr string) (bool, error)

func InitEmpty 

func InitEmpty(ctx context.Context, repo drivers.RepoStore, instanceID, title string) error

InitEmpty initializes an empty project

func IsInit 

func IsInit(ctx context.Context, repo drivers.RepoStore, instanceID string) bool

IsInit returns true if a Rill project exists in the repo

func ResolveTemplate 

func ResolveTemplate(tmpl string, data TemplateData) (string, error)

ResolveTemplate resolves a template to a string using the given data.

func ResolveTemplateRecursively added in v0.48.0 

func ResolveTemplateRecursively(val any, data TemplateData) (any, error)

ResolveTemplateRecursively recursively traverses the provided value and applies ResolveTemplate to any string it encounters. It may overwrite the provided value in-place.

Types

type APIYAML added in v0.42.0 

type APIYAML struct {
	DataYAML `yaml:",inline" mapstructure:",squash"`
	OpenAPI  *OpenAPIYAML `yaml:"openapi"`
}

APIYAML is the raw structure of a API resource defined in YAML (does not include common fields)

type AlertYAML added in v0.41.0 

type AlertYAML struct {
	Title     string        `yaml:"title"`
	Refresh   *ScheduleYAML `yaml:"refresh"`
	Watermark string        `yaml:"watermark"` // options: "trigger_time", "inherit"
	Intervals struct {
		Duration      string `yaml:"duration"`
		Limit         uint   `yaml:"limit"`
		CheckUnclosed bool   `yaml:"check_unclosed"`
	} `yaml:"intervals"`
	Timeout string    `yaml:"timeout"`
	Data    *DataYAML `yaml:"data"`
	For     struct {
		UserID     string         `yaml:"user_id"`
		UserEmail  string         `yaml:"user_email"`
		Attributes map[string]any `yaml:"attributes"`
	} `yaml:"for"`
	Query struct {
		Name     string         `yaml:"name"`
		Args     map[string]any `yaml:"args"`
		ArgsJSON string         `yaml:"args_json"`
		For      struct {
			UserID     string         `yaml:"user_id"`
			UserEmail  string         `yaml:"user_email"`
			Attributes map[string]any `yaml:"attributes"`
		} `yaml:"for"`
	} `yaml:"query"`
	OnRecover     *bool  `yaml:"on_recover"`
	OnFail        *bool  `yaml:"on_fail"`
	OnError       *bool  `yaml:"on_error"`
	Renotify      *bool  `yaml:"renotify"`
	RenotifyAfter string `yaml:"renotify_after"`
	Notify        struct {
		Email struct {
			Recipients []string `yaml:"recipients"`
		} `yaml:"email"`
		Slack struct {
			Users    []string `yaml:"users"`
			Channels []string `yaml:"channels"`
			Webhooks []string `yaml:"webhooks"`
		} `yaml:"slack"`
	} `yaml:"notify"`
	Annotations map[string]string `yaml:"annotations"`
	// Backwards compatibility
	Email struct {
		Recipients    []string `yaml:"recipients"`
		OnRecover     *bool    `yaml:"on_recover"`
		OnFail        *bool    `yaml:"on_fail"`
		OnError       *bool    `yaml:"on_error"`
		Renotify      *bool    `yaml:"renotify"`
		RenotifyAfter string   `yaml:"renotify_after"`
	} `yaml:"email"`
	// contains filtered or unexported fields
}

AlertYAML is the raw structure of an Alert resource defined in YAML (does not include common fields)

type AvailableComparisonOffset added in v0.38.0 

type AvailableComparisonOffset struct {
	Offset string
	Range  string
}

func (*AvailableComparisonOffset) UnmarshalYAML added in v0.38.0 

func (o *AvailableComparisonOffset) UnmarshalYAML(v *yaml.Node) error

type AvailableTimeRange added in v0.38.0 

type AvailableTimeRange struct {
	Range             string
	ComparisonOffsets []AvailableComparisonOffset
}

func (*AvailableTimeRange) UnmarshalYAML added in v0.38.0 

func (t *AvailableTimeRange) UnmarshalYAML(v *yaml.Node) error

type ComponentYAML added in v0.44.0 

type ComponentYAML struct {
	Title    string                    `yaml:"title"`
	Subtitle string                    `yaml:"subtitle"`
	Data     *DataYAML                 `yaml:"data"`
	VegaLite *string                   `yaml:"vega_lite"`
	Other    map[string]map[string]any `yaml:",inline" mapstructure:",remain"` // Generic renderer: can only have one key
	// contains filtered or unexported fields
}

type Connector 

type Connector struct {
	Name            string
	Driver          string
	Spec            *drivers.Spec
	DefaultConfig   map[string]string
	Resources       []*Resource
	AnonymousAccess bool
	Err             error
}

Connector contains metadata about a connector used in a Rill project

type ConnectorDef 

type ConnectorDef struct {
	Type     string
	Name     string
	Defaults map[string]string
}

ConnectorDef is a subtype of RillYAML, defining connectors required by the project

type ConnectorYAML added in v0.46.0 

type ConnectorYAML struct {

	// Driver name
	Driver   string            `yaml:"driver"`
	Defaults map[string]string `yaml:",inline" mapstructure:",remain"`
	// contains filtered or unexported fields
}

ConnectorYAML is the raw structure of a Connector resource defined in YAML (does not include common fields)

type DashboardYAML added in v0.42.0 

type DashboardYAML struct {
	Title   string `yaml:"title"`
	Columns uint32 `yaml:"columns"`
	Gap     uint32 `yaml:"gap"`
	Items   []*struct {
		Component yaml.Node `yaml:"component"` // Can be a name (string) or inline component definition (map)
		X         *uint32   `yaml:"x"`
		Y         *uint32   `yaml:"y"`
		Width     *uint32   `yaml:"width"`
		Height    *uint32   `yaml:"height"`
	} `yaml:"items"`
	// contains filtered or unexported fields
}

type DataYAML added in v0.42.0 

type DataYAML struct {
	Connector  string         `yaml:"connector"`
	SQL        string         `yaml:"sql"`
	MetricsSQL string         `yaml:"metrics_sql"`
	API        string         `yaml:"api"`
	Args       map[string]any `yaml:"args"`
	Glob       yaml.Node      `yaml:"glob"` // Path (string) or properties (map[string]any)
}

DataYAML is the raw YAML structure of a sub-property for defining a data resolver and properties. It is used across multiple resources, usually under "data:", but inlined for APIs.

type Diff 

type Diff struct {
	Reloaded       bool
	Skipped        bool
	Added          []ResourceName
	Modified       []ResourceName
	ModifiedDotEnv bool
	Deleted        []ResourceName
}

Diff shows changes to Parser.Resources following an incremental reparse.

type MetricsViewFieldSelectorYAML added in v0.47.0 

type MetricsViewFieldSelectorYAML struct {
	Name       string
	TimeGrain  runtimev1.TimeGrain // Only for time dimensions
	Descending bool                // Only for sorting
}

func (*MetricsViewFieldSelectorYAML) UnmarshalYAML added in v0.47.0 

func (f *MetricsViewFieldSelectorYAML) UnmarshalYAML(v *yaml.Node) error

type MetricsViewFieldSelectorsYAML added in v0.47.0 

type MetricsViewFieldSelectorsYAML []MetricsViewFieldSelectorYAML

func (*MetricsViewFieldSelectorsYAML) UnmarshalYAML added in v0.47.0 

func (f *MetricsViewFieldSelectorsYAML) UnmarshalYAML(v *yaml.Node) error

type MetricsViewMeasureWindow added in v0.47.0 

type MetricsViewMeasureWindow struct {
	Partition bool
	Order     []MetricsViewFieldSelectorYAML
	OrderTime bool // Preset for ordering by only the time dimension
	Frame     string
}

func (*MetricsViewMeasureWindow) UnmarshalYAML added in v0.47.0 

func (f *MetricsViewMeasureWindow) UnmarshalYAML(v *yaml.Node) error

type MetricsViewSecurityPolicyYAML added in v0.47.0 

type MetricsViewSecurityPolicyYAML struct {
	Access    string `yaml:"access"`
	RowFilter string `yaml:"row_filter"`
	Include   []*struct {
		Condition string    `yaml:"if"`
		Names     yaml.Node // []string or "*" (will be parsed with parseNamesYAML)
	}
	Exclude []*struct {
		Condition string    `yaml:"if"`
		Names     yaml.Node // []string or "*" (will be parsed with parseNamesYAML)
	}
	Rules []*MetricsViewSecurityRuleYAML `yaml:"rules"`
}

func (*MetricsViewSecurityPolicyYAML) Proto added in v0.47.0 

func (p *MetricsViewSecurityPolicyYAML) Proto() ([]*runtimev1.SecurityRule, error)

type MetricsViewSecurityRuleYAML added in v0.47.0 

type MetricsViewSecurityRuleYAML struct {
	Type   string
	Action string
	If     string
	Names  []string
	All    bool
	SQL    string
}

func (*MetricsViewSecurityRuleYAML) Proto added in v0.47.0 

func (r *MetricsViewSecurityRuleYAML) Proto() (*runtimev1.SecurityRule, error)

type MetricsViewYAML added in v0.37.0 

type MetricsViewYAML struct {
	Title              string   `yaml:"title"`
	DisplayName        string   `yaml:"display_name"` // Backwards compatibility
	Description        string   `yaml:"description"`
	Model              string   `yaml:"model"`
	Database           string   `yaml:"database"`
	DatabaseSchema     string   `yaml:"database_schema"`
	Table              string   `yaml:"table"`
	TimeDimension      string   `yaml:"timeseries"`
	Watermark          string   `yaml:"watermark"`
	SmallestTimeGrain  string   `yaml:"smallest_time_grain"`
	DefaultTimeRange   string   `yaml:"default_time_range"`
	AvailableTimeZones []string `yaml:"available_time_zones"`
	FirstDayOfWeek     uint32   `yaml:"first_day_of_week"`
	FirstMonthOfYear   uint32   `yaml:"first_month_of_year"`
	DefaultTheme       string   `yaml:"default_theme"`
	Dimensions         []*struct {
		Name        string
		Label       string
		Column      string
		Expression  string
		Property    string // For backwards compatibility
		Description string
		Ignore      bool `yaml:"ignore"` // Deprecated
		Unnest      bool
		URI         string
	}
	DefaultDimensions []string `yaml:"default_dimensions"`
	Measures          []*struct {
		Name                string
		Label               string
		Type                string
		Expression          string
		Window              *MetricsViewMeasureWindow
		Per                 MetricsViewFieldSelectorsYAML
		Requires            MetricsViewFieldSelectorsYAML
		Description         string
		FormatPreset        string `yaml:"format_preset"`
		FormatD3            string `yaml:"format_d3"`
		Ignore              bool   `yaml:"ignore"` // Deprecated
		ValidPercentOfTotal bool   `yaml:"valid_percent_of_total"`
	}
	DefaultMeasures   []string `yaml:"default_measures"`
	Security          *MetricsViewSecurityPolicyYAML
	DefaultComparison struct {
		Mode      string `yaml:"mode"`
		Dimension string `yaml:"dimension"`
	} `yaml:"default_comparison"`
	AvailableTimeRanges []AvailableTimeRange `yaml:"available_time_ranges"`
	// contains filtered or unexported fields
}

MetricsViewYAML is the raw structure of a MetricsView resource defined in YAML

type MigrationYAML added in v0.37.0 

type MigrationYAML struct {
	Version uint `yaml:"version" mapstructure:"version"`
}

MigrationYAML is the raw structure of a Migration resource defined in YAML (does not include common fields)

type ModelYAML added in v0.37.0 

type ModelYAML struct {
	Refresh           *ScheduleYAML  `yaml:"refresh"`
	Timeout           string         `yaml:"timeout"`
	Incremental       bool           `yaml:"incremental"`
	State             *DataYAML      `yaml:"state"`
	Splits            *DataYAML      `yaml:"splits"`
	SplitsWatermark   string         `yaml:"splits_watermark"`
	SplitsConcurrency uint           `yaml:"splits_concurrency"`
	InputProperties   map[string]any `yaml:",inline" mapstructure:",remain"`
	Stage             struct {
		Connector  string         `yaml:"connector"`
		Properties map[string]any `yaml:",inline" mapstructure:",remain"`
	} `yaml:"stage"`
	Output struct {
		Connector  string         `yaml:"connector"`
		Properties map[string]any `yaml:",inline" mapstructure:",remain"`
	} `yaml:"output"`
	Materialize *bool `yaml:"materialize"`
	// contains filtered or unexported fields
}

ModelYAML is the raw structure of a Model resource defined in YAML (does not include common fields)

type Node 

type Node struct {
	Kind              ResourceKind
	Name              string
	Refs              []ResourceName
	Paths             []string
	YAML              *yaml.Node
	YAMLOverride      *yaml.Node
	YAMLRaw           string
	YAMLPath          string
	Connector         string
	ConnectorInferred bool
	SQL               string
	SQLPath           string
	SQLAnnotations    map[string]any
	SQLUsesTemplating bool
}

Node represents one path stem in the project. It contains data derived from a YAML and/or SQL file (e.g. "/path/to/file.yaml" for "/path/to/file.sql").

type OpenAPIYAML added in v0.48.0 

type OpenAPIYAML struct {
	Summary string `yaml:"summary"`
	Request struct {
		Parameters []map[string]any `yaml:"parameters"`
	} `yaml:"request"`
	Response struct {
		Schema map[string]any `yaml:"schema"`
	} `yaml:"response"`
}

type Parser 

type Parser struct {
	// Options
	Repo                 drivers.RepoStore
	InstanceID           string
	Environment          string
	DefaultOLAPConnector string

	// Output
	RillYAML  *RillYAML
	DotEnv    map[string]string
	Resources map[ResourceName]*Resource
	Errors    []*runtimev1.ParseError
	// contains filtered or unexported fields
}

Parser parses a Rill project directory into a set of resources. After the initial parse, the parser can be used to incrementally reparse a subset of files. Parser is not concurrency safe.

func Parse 

func Parse(ctx context.Context, repo drivers.RepoStore, instanceID, environment, defaultOLAPConnector string) (*Parser, error)

Parse creates a new parser and parses the entire project.

func (*Parser) AnalyzeConnectors 

func (p *Parser) AnalyzeConnectors(ctx context.Context) []*Connector

AnalyzeConnectors extracts connector metadata from a Rill project

func (*Parser) IsSkippable added in v0.44.0 

func (p *Parser) IsSkippable(path string) bool

IsSkippable returns true if the path will be skipped by Reparse. It's useful for callers to avoid triggering a reparse when they know the path is not relevant.

func (*Parser) Reparse 

func (p *Parser) Reparse(ctx context.Context, paths []string) (*Diff, error)

Reparse re-parses the indicated file paths, updating the Parser's state. If rill.yaml has previously errored, or if rill.yaml is included in paths, it will reload the entire project. If a previous call to Reparse has returned an error, the Parser may not be accessed or called again.

func (*Parser) TrackedPathsInDir added in v0.44.0 

func (p *Parser) TrackedPathsInDir(dir string) []string

TrackedPathsInDir returns the paths under the given directory that the parser currently has cached results for.

type ReportYAML added in v0.37.0 

type ReportYAML struct {
	Title     string        `yaml:"title"`
	Refresh   *ScheduleYAML `yaml:"refresh"`
	Watermark string        `yaml:"watermark"` // options: "trigger_time", "inherit"
	Intervals struct {
		Duration      string `yaml:"duration"`
		Limit         uint   `yaml:"limit"`
		CheckUnclosed bool   `yaml:"check_unclosed"`
	} `yaml:"intervals"`
	Timeout string `yaml:"timeout"`
	Query   struct {
		Name     string         `yaml:"name"`
		Args     map[string]any `yaml:"args"`
		ArgsJSON string         `yaml:"args_json"`
	} `yaml:"query"`
	Export struct {
		Format string `yaml:"format"`
		Limit  uint   `yaml:"limit"`
	} `yaml:"export"`
	Email struct {
		Recipients []string `yaml:"recipients"`
	} `yaml:"email"`
	Notify struct {
		Email struct {
			Recipients []string `yaml:"recipients"`
		} `yaml:"email"`
		Slack struct {
			Users    []string `yaml:"users"`
			Channels []string `yaml:"channels"`
			Webhooks []string `yaml:"webhooks"`
		} `yaml:"slack"`
	} `yaml:"notify"`
	Annotations map[string]string `yaml:"annotations"`
	// contains filtered or unexported fields
}

ReportYAML is the raw structure of a Report resource defined in YAML (does not include common fields)

type Resource 

type Resource struct {
	// Metadata
	Name  ResourceName
	Paths []string
	Refs  []ResourceName // Derived from rawRefs after parsing (can't contain ResourceKindUnspecified). Always sorted.

	// Only one of these will be non-nil
	SourceSpec      *runtimev1.SourceSpec
	ModelSpec       *runtimev1.ModelSpec
	MetricsViewSpec *runtimev1.MetricsViewSpec
	MigrationSpec   *runtimev1.MigrationSpec
	ReportSpec      *runtimev1.ReportSpec
	AlertSpec       *runtimev1.AlertSpec
	ThemeSpec       *runtimev1.ThemeSpec
	ComponentSpec   *runtimev1.ComponentSpec
	DashboardSpec   *runtimev1.DashboardSpec
	APISpec         *runtimev1.APISpec
	ConnectorSpec   *runtimev1.ConnectorSpec
	// contains filtered or unexported fields
}

Resource parsed from code files. One file may output multiple resources and multiple files may contribute config to one resource.

type ResourceKind 

type ResourceKind int

ResourceKind identifies a resource type supported by the parser 

const (
	ResourceKindUnspecified ResourceKind = iota
	ResourceKindSource
	ResourceKindModel
	ResourceKindMetricsView
	ResourceKindMigration
	ResourceKindReport
	ResourceKindAlert
	ResourceKindTheme
	ResourceKindComponent
	ResourceKindDashboard
	ResourceKindAPI
	ResourceKindConnector
)

func ParseResourceKind 

func ParseResourceKind(kind string) (ResourceKind, error)

ParseResourceKind maps a string to a ResourceKind. Note: The empty string is considered a valid kind (unspecified).

func (ResourceKind) String 

func (k ResourceKind) String() string

type ResourceName 

type ResourceName struct {
	Kind ResourceKind
	Name string
}

ResourceName is a unique identifier for a resource

func (ResourceName) Normalized 

func (n ResourceName) Normalized() ResourceName

func (ResourceName) String 

func (n ResourceName) String() string

type RillYAML 

type RillYAML struct {
	Title         string
	Description   string
	OLAPConnector string
	Connectors    []*ConnectorDef
	Variables     []*VariableDef
	Defaults      map[ResourceKind]yaml.Node
	FeatureFlags  map[string]bool
	PublicPaths   []string
}

RillYAML is the parsed contents of rill.yaml

func ParseRillYAML 

func ParseRillYAML(ctx context.Context, repo drivers.RepoStore, instanceID string) (*RillYAML, error)

ParseRillYAML parses only the project's rill.yaml (or rill.yml) file.

type ScheduleYAML added in v0.37.0 

type ScheduleYAML struct {
	RefUpdate *bool  `yaml:"ref_update" mapstructure:"ref_update"`
	Cron      string `yaml:"cron" mapstructure:"cron"`
	Every     string `yaml:"every" mapstructure:"every"`
	TimeZone  string `yaml:"time_zone" mapstructure:"time_zone"`
	Disable   bool   `yaml:"disable" mapstructure:"disable"`
}

ScheduleYAML is the raw structure of a refresh schedule clause defined in YAML. This does not represent a stand-alone YAML file, just a partial used in other structs.

type SourceYAML added in v0.37.0 

type SourceYAML struct {
	Timeout    string         `yaml:"timeout"`
	Refresh    *ScheduleYAML  `yaml:"refresh"`
	Properties map[string]any `yaml:",inline" mapstructure:",remain"`
	// contains filtered or unexported fields
}

SourceYAML is the raw structure of a Source resource defined in YAML (does not include common fields)

type TemplateData 

type TemplateData struct {
	Environment string
	User        map[string]any
	Variables   map[string]string
	State       map[string]any
	ExtraProps  map[string]any
	Self        TemplateResource
	Resolve     func(ref ResourceName) (string, error)
	Lookup      func(name ResourceName) (TemplateResource, error)
}

TemplateData contains data for resolving a template.

type TemplateMetadata 

type TemplateMetadata struct {
	Refs                     []ResourceName
	Config                   map[string]any
	Variables                []string
	UsesTemplating           bool
	ResolvedWithPlaceholders string
}

TemplateMetadata contains metadata extracted from a template.

func AnalyzeTemplate 

func AnalyzeTemplate(tmpl string) (*TemplateMetadata, error)

AnalyzeTemplate parses a template and extracts metadata.

type TemplateResource 

type TemplateResource struct {
	Meta  *runtimev1.ResourceMeta
	Spec  any
	State any
}

TemplateResource contains data for a resource for injection into a template.

type ThemeYAML added in v0.38.0 

type ThemeYAML struct {
	Colors struct {
		Primary   string `yaml:"primary"`
		Secondary string `yaml:"secondary"`
	} `yaml:"colors"`
	// contains filtered or unexported fields
}

ThemeYAML is the raw structure of a Theme for the UI in YAML (does not include common fields)

type VariableDef 

type VariableDef struct {
	Name    string
	Default string
}

VariableDef is a subtype of RillYAML, defining defaults for project variables

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.