drivers

Documentation

Index

• Constants
• Variables
• func Register(name string, driver Driver)
• type CatalogObject
• type CatalogStore
• type Column
• type Connection
  • func Open(driver string, dsn string) (Connection, error)
• type Driver
• type InformationSchema
• type Instance
• type OLAPStore
• type RegistryStore
• type Repo
• type RepoStore
• type Statement
• type Table

Constants

const (
	CatalogObjectTypeSource         string = "source"
	CatalogObjectTypeMetricsView    string = "metrics_view"
	CatalogObjectTypeUnmanagedTable string = "unmanaged_table"
)

Constants representing different kinds of catalog objects

Variables

var Drivers = make(map[string]Driver)

Drivers is a registry of drivers

var ErrClosed = errors.New("driver: connection is closed")

ErrClosed indicates the connection is closed

var ErrNotFound = errors.New("driver: not found")

ErrNotFound indicates the resource wasn't found

var ErrUnsupportedConnector = errors.New("drivers: connector not supported")

ErrUnsupportedConnector is returned from Ingest for unsupported connectors

Functions

func Register

func Register(name string, driver Driver)

Register registers a new driver

Types

type CatalogObject

type CatalogObject struct {
	Name      string
	Type      string
	SQL       string
	CreatedOn time.Time `db:"created_on"`
	UpdatedOn time.Time `db:"updated_on"`
}

CatalogObject represents one object in the catalog, such as a source

type CatalogStore

type CatalogStore interface {
	FindObjects(ctx context.Context, instanceID string) []*CatalogObject
	FindObject(ctx context.Context, instanceID string, name string) (*CatalogObject, bool)
	CreateObject(ctx context.Context, instanceID string, object *CatalogObject) error
	UpdateObject(ctx context.Context, instanceID string, object *CatalogObject) error
	DeleteObject(ctx context.Context, instanceID string, name string) error
}

CatalogStore is implemented by drivers capable of storing catalog info for a specific instance

type Column

type Column struct {
	Name     string
	Type     string
	Nullable bool
}

Column represents a column in a table

type Connection

type Connection interface {
	// Migrate prepares the connection for use. It will be called before the connection is first used.
	// (Not to be confused with migrating artifacts, which is handled by the runtime and tracked in the catalog.)
	Migrate(ctx context.Context) error

	// MigrationStatus returns the connection's current and desired migration version (if applicable)
	MigrationStatus(ctx context.Context) (current int, desired int, err error)

	// Close closes the connection
	Close() error

	// RegistryStore returns a RegistryStore if the driver can serve as such, otherwise returns false.
	// The registry is responsible for tracking instances and repos.
	RegistryStore() (RegistryStore, bool)

	// CatalogStore returns a CatalogStore if the driver can serve as such, otherwise returns false.
	// A catalog is used to store state about migrated/deployed objects (such as sources and metrics views).
	CatalogStore() (CatalogStore, bool)

	// RepoStore returns a RepoStore if the driver can serve as such, otherwise returns false.
	// A repo stores file artifacts (either in a folder or virtualized in a database).
	RepoStore() (RepoStore, bool)

	// OLAPStore returns an OLAPStore if the driver can serve as such, otherwise returns false.
	// OLAP stores are where we actually store, transform, and query users' data.
	OLAPStore() (OLAPStore, bool)
}

Connection represents a connection to an underlying DB. It should implement one or more of RegistryStore, CatalogStore, RepoStore, and OLAPStore.

func Open

func Open(driver string, dsn string) (Connection, error)

Open opens a new connection

type Driver

type Driver interface {
	Open(dsn string) (Connection, error)
}

Driver represents an underlying DB

type InformationSchema

type InformationSchema interface {
	All(ctx context.Context) ([]*Table, error)
	Lookup(ctx context.Context, name string) (*Table, error)
}

InformationSchema contains information about existing tables in an OLAP driver

type Instance

type Instance struct {
	// Identifier
	ID string
	// Driver is the driver of the OLAP store to connect to ("druid" or "duckdb" currently)
	Driver string
	// DSN is the connection string for the OLAP store
	DSN string
	// ObjectPrefix will be prepended to tables and views created in the OLAP store through Rill SQL.
	// You can view it as a simpler alternative to using different database schemas.
	ObjectPrefix string `db:"object_prefix"`
	// Exposed indicates that the underlying OLAP infra may be manipulated directly by users.
	// If true, the runtime will continuously poll the infra's information schema to discover tables not created through the runtime.
	Exposed bool
	// EmbedCatalog tells the runtime whether to store the instance's catalog data (such as sources and metrics views)
	// directly in the OLAP datastore instead of in the runtime's metadata store.
	EmbedCatalog bool `db:"embed_catalog"`
	// CreatedOn is when the instance was created
	CreatedOn time.Time `db:"created_on"`
	// UpdatedOn is when the instance was last updated in the registry
	UpdatedOn time.Time `db:"updated_on"`
}

Instance represents one deployment of an OLAP store

type OLAPStore

type OLAPStore interface {
	Execute(ctx context.Context, stmt *Statement) (*sqlx.Rows, error)
	Ingest(ctx context.Context, source *connectors.Source) error
	InformationSchema() InformationSchema
}

OLAPStore is implemented by drivers that are capable of storing, transforming and serving analytical queries

type RegistryStore

type RegistryStore interface {
	FindInstances(ctx context.Context) []*Instance
	FindInstance(ctx context.Context, id string) (*Instance, bool)
	CreateInstance(ctx context.Context, instance *Instance) error
	DeleteInstance(ctx context.Context, id string) error
	FindRepos(ctx context.Context) []*Repo
	FindRepo(ctx context.Context, id string) (*Repo, bool)
	CreateRepo(ctx context.Context, repo *Repo) error
	DeleteRepo(ctx context.Context, id string) error
}

RegistryStore is implemented by drivers capable of storing and looking up instances and repos

type Repo

type Repo struct {
	ID        string
	Driver    string
	DSN       string
	CreatedOn time.Time `db:"created_on"`
	UpdatedOn time.Time `db:"updated_on"`
}

Repo represents a file artifact store (either a folder on disk or virtualized in a database)

type RepoStore

type RepoStore interface {
	ListRecursive(ctx context.Context, repoID string) ([]string, error)
	Get(ctx context.Context, repoID string, path string) (string, error)
	Put(ctx context.Context, repoID string, path string, blob string) error
	Delete(ctx context.Context, repoID string, path string) error
}

RepoStore is implemented by drivers capable of storing SQL file artifacts

type Statement

type Statement struct {
	Query    string
	Args     []any
	DryRun   bool
	Priority int
}

Statement wraps a query to execute against an OLAP driver

type Table

type Table struct {
	Database string
	Schema   string
	Name     string
	Type     string
	Columns  []Column
}

Table represents a table in an information schema