drivers

Documentation

Index

• Variables
• func Register(name string, driver Driver)
• type CatalogObject
• type CatalogObjectType
• type CatalogStore
• type Connection
  • func Open(driver string, dsn string) (Connection, error)
• type Driver
• type InformationSchema
• type Instance
• type OLAPStore
• type RegistryStore
• type Repo
• type RepoObjectStat
• type RepoStore
• type Result
• type Statement
• type Table

Variables

var Drivers = make(map[string]Driver)

Drivers is a registry of drivers

var ErrClosed = errors.New("driver: connection is closed")

ErrClosed indicates the connection is closed

var ErrNotFound = errors.New("driver: not found")

ErrNotFound indicates the resource wasn't found

var ErrUnsupportedConnector = errors.New("drivers: connector not supported")

ErrUnsupportedConnector is returned from Ingest for unsupported connectors

Functions

func Register

func Register(name string, driver Driver)

Register registers a new driver

Types

type CatalogObject

type CatalogObject struct {
	// basic fields
	Name    string
	Type    CatalogObjectType
	SQL     string
	Schema  *api.StructType
	Managed bool

	// artifact fields
	Definition []byte
	Path       string

	CreatedOn   time.Time `db:"created_on"`
	UpdatedOn   time.Time `db:"updated_on"`
	RefreshedOn time.Time `db:"refreshed_on"`
}

CatalogObject represents one object in the catalog, such as a source

type CatalogObjectType

type CatalogObjectType string

const (
	CatalogObjectTypeUnspecified CatalogObjectType = ""
	CatalogObjectTypeTable       CatalogObjectType = "table"
	CatalogObjectTypeSource      CatalogObjectType = "source"
	CatalogObjectTypeModel       CatalogObjectType = "model"
	CatalogObjectTypeMetricsView CatalogObjectType = "metrics_view"
)

type CatalogStore

type CatalogStore interface {
	FindObjects(ctx context.Context, instanceID string, typ CatalogObjectType) []*CatalogObject
	FindObject(ctx context.Context, instanceID string, name string) (*CatalogObject, bool)
	CreateObject(ctx context.Context, instanceID string, object *CatalogObject) error
	UpdateObject(ctx context.Context, instanceID string, object *CatalogObject) error
	DeleteObject(ctx context.Context, instanceID string, name string) error
}

CatalogStore is implemented by drivers capable of storing catalog info for a specific instance

type Connection

type Connection interface {
	// Migrate prepares the connection for use. It will be called before the connection is first used.
	// (Not to be confused with migrating artifacts, which is handled by the runtime and tracked in the catalog.)
	Migrate(ctx context.Context) error

	// MigrationStatus returns the connection's current and desired migration version (if applicable)
	MigrationStatus(ctx context.Context) (current int, desired int, err error)

	// Close closes the connection
	Close() error

	// RegistryStore returns a RegistryStore if the driver can serve as such, otherwise returns false.
	// The registry is responsible for tracking instances and repos.
	RegistryStore() (RegistryStore, bool)

	// CatalogStore returns a CatalogStore if the driver can serve as such, otherwise returns false.
	// A catalog is used to store state about migrated/deployed objects (such as sources and metrics views).
	CatalogStore() (CatalogStore, bool)

	// RepoStore returns a RepoStore if the driver can serve as such, otherwise returns false.
	// A repo stores file artifacts (either in a folder or virtualized in a database).
	RepoStore() (RepoStore, bool)

	// OLAPStore returns an OLAPStore if the driver can serve as such, otherwise returns false.
	// OLAP stores are where we actually store, transform, and query users' data.
	OLAPStore() (OLAPStore, bool)
}

Connection represents a connection to an underlying DB. It should implement one or more of RegistryStore, CatalogStore, RepoStore, and OLAPStore.

func Open

func Open(driver string, dsn string) (Connection, error)

Open opens a new connection

type Driver

type Driver interface {
	Open(dsn string) (Connection, error)
}

Driver represents an underlying DB

type InformationSchema

type InformationSchema interface {
	All(ctx context.Context) ([]*Table, error)
	Lookup(ctx context.Context, name string) (*Table, error)
}

InformationSchema contains information about existing tables in an OLAP driver

type Instance

type Instance struct {
	// Identifier
	ID string
	// Driver is the driver of the OLAP store to connect to ("druid" or "duckdb" currently)
	Driver string
	// DSN is the connection string for the OLAP store
	DSN string
	// ObjectPrefix will be prepended to tables and views created in the OLAP store through Rill SQL.
	// You can view it as a simpler alternative to using different database schemas.
	ObjectPrefix string `db:"object_prefix"`
	// Exposed indicates that the underlying OLAP infra may be manipulated directly by users.
	// If true, the runtime will continuously poll the infra's information schema to discover tables not created through the runtime.
	Exposed bool
	// EmbedCatalog tells the runtime whether to store the instance's catalog data (such as sources and metrics views)
	// directly in the OLAP datastore instead of in the runtime's metadata store.
	EmbedCatalog bool `db:"embed_catalog"`
	// CreatedOn is when the instance was created
	CreatedOn time.Time `db:"created_on"`
	// UpdatedOn is when the instance was last updated in the registry
	UpdatedOn time.Time `db:"updated_on"`
}

Instance represents one deployment of an OLAP store

type OLAPStore

type OLAPStore interface {
	Execute(ctx context.Context, stmt *Statement) (*Result, error)
	Ingest(ctx context.Context, env *connectors.Env, source *connectors.Source) error
	InformationSchema() InformationSchema
}

OLAPStore is implemented by drivers that are capable of storing, transforming and serving analytical queries

type RegistryStore

type RegistryStore interface {
	FindInstances(ctx context.Context) []*Instance
	FindInstance(ctx context.Context, id string) (*Instance, bool)
	CreateInstance(ctx context.Context, instance *Instance) error
	DeleteInstance(ctx context.Context, id string) error
	FindRepos(ctx context.Context) []*Repo
	FindRepo(ctx context.Context, id string) (*Repo, bool)
	CreateRepo(ctx context.Context, repo *Repo) error
	DeleteRepo(ctx context.Context, id string) error
}

RegistryStore is implemented by drivers capable of storing and looking up instances and repos

type Repo

type Repo struct {
	ID        string
	Driver    string
	DSN       string
	CreatedOn time.Time `db:"created_on"`
	UpdatedOn time.Time `db:"updated_on"`
}

Repo represents a file artifact store (either a folder on disk or virtualized in a database)

type RepoObjectStat

type RepoObjectStat struct {
	LastUpdated time.Time
}

type RepoStore

type RepoStore interface {
	Driver() string
	DSN() string
	ListRecursive(ctx context.Context, repoID string, glob string) ([]string, error)
	Get(ctx context.Context, repoID string, path string) (string, error)
	Stat(ctx context.Context, repoID string, path string) (*RepoObjectStat, error)
	PutBlob(ctx context.Context, repoID string, path string, blob string) error
	PutReader(ctx context.Context, repoID string, path string, reader io.Reader) (string, error)
	Rename(ctx context.Context, repoID string, from string, path string) error
	Delete(ctx context.Context, repoID string, path string) error
}

RepoStore is implemented by drivers capable of storing SQL file artifacts

type Result

type Result struct {
	*sqlx.Rows
	Schema *api.StructType
}

Result wraps the results of query

type Statement

type Statement struct {
	Query    string
	Args     []any
	DryRun   bool
	Priority int
}

Statement wraps a query to execute against an OLAP driver

type Table

type Table struct {
	Database       string
	DatabaseSchema string
	Name           string
	Schema         *api.StructType
}

Table represents a table in an information schema