bigquery

Documentation

Index

• func NewMaterializer(fullRefresh bool) *pipeline.Materializer
• type AcceptedValuesCheck
  • func (c *AcceptedValuesCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error
• type BasicOperator
  • func NewBasicOperator(conn connectionFetcher, extractor queryExtractor, materializer materializer) *BasicOperator
  • func (o BasicOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
  • func (o BasicOperator) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error
• type Client
  • func NewDB(c *Config) (*Client, error)
  • func (d *Client) GetIngestrURI() (string, error)
  • func (d *Client) IsValid(ctx context.Context, query *query.Query) (bool, error)
  • func (d *Client) RunQueryWithoutResult(ctx context.Context, query *query.Query) error
  • func (d *Client) Select(ctx context.Context, query *query.Query) ([][]interface{}, error)
  • func (d *Client) SelectWithSchema(ctx context.Context, queryObj *query.Query) (*query.QueryResult, error)
  • func (d *Client) Test(ctx context.Context) error
  • func (d *Client) UpdateTableMetadataIfNotExist(ctx context.Context, asset *pipeline.Asset) error
• type ColumnCheckOperator
  • func NewColumnCheckOperator(manager connectionFetcher) (*ColumnCheckOperator, error)
  • func (o ColumnCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
• type Config
  • func (c Config) GetConnectionURI() (string, error)
  • func (c Config) GetIngestrURI() (string, error)
  • func (c Config) IsValid() bool
• type DB
• type MetadataPushOperator
  • func NewMetadataPushOperator(conn connectionFetcher) *MetadataPushOperator
  • func (o *MetadataPushOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
• type MetadataUpdater
• type NoMetadataUpdatedError
  • func (m NoMetadataUpdatedError) Error() string
• type PatternCheck
  • func (c *PatternCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error
• type Querier
• type Selector

Functions

func NewMaterializer

func NewMaterializer(fullRefresh bool) *pipeline.Materializer

Types

type AcceptedValuesCheck

type AcceptedValuesCheck struct {
	// contains filtered or unexported fields
}

func (*AcceptedValuesCheck) Check

func (c *AcceptedValuesCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type BasicOperator

type BasicOperator struct {
	// contains filtered or unexported fields
}

func NewBasicOperator

func NewBasicOperator(conn connectionFetcher, extractor queryExtractor, materializer materializer) *BasicOperator

func (BasicOperator) Run

func (o BasicOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

func (BasicOperator) RunTask

func (o BasicOperator) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error

type Client

type Client struct {
	// contains filtered or unexported fields
}

func NewDB

func NewDB(c *Config) (*Client, error)

func (*Client) GetIngestrURI

func (d *Client) GetIngestrURI() (string, error)

func (*Client) IsValid

func (d *Client) IsValid(ctx context.Context, query *query.Query) (bool, error)

func (*Client) RunQueryWithoutResult

func (d *Client) RunQueryWithoutResult(ctx context.Context, query *query.Query) error

func (*Client) Select

func (d *Client) Select(ctx context.Context, query *query.Query) ([][]interface{}, error)

func (*Client) SelectWithSchema

func (d *Client) SelectWithSchema(ctx context.Context, queryObj *query.Query) (*query.QueryResult, error)

func (*Client) Test

func (d *Client) Test(ctx context.Context) error

Test runs a simple query (SELECT 1) to validate the connection.

func (*Client) UpdateTableMetadataIfNotExist

func (d *Client) UpdateTableMetadataIfNotExist(ctx context.Context, asset *pipeline.Asset) error

type ColumnCheckOperator

type ColumnCheckOperator struct {
	// contains filtered or unexported fields
}

func NewColumnCheckOperator

func NewColumnCheckOperator(manager connectionFetcher) (*ColumnCheckOperator, error)

func (ColumnCheckOperator) Run

func (o ColumnCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

type Config

type Config struct {
	ProjectID           string `envconfig:"BIGQUERY_PROJECT"`
	CredentialsFilePath string `envconfig:"BIGQUERY_CREDENTIALS_FILE"`
	CredentialsJSON     string
	Credentials         *google.Credentials
	Location            string `envconfig:"BIGQUERY_LOCATION"`
}

func (Config) GetConnectionURI

func (c Config) GetConnectionURI() (string, error)

func (Config) GetIngestrURI

func (c Config) GetIngestrURI() (string, error)

func (Config) IsValid

func (c Config) IsValid() bool

type DB

type DB interface {
	Querier
	Selector
	MetadataUpdater
}

type MetadataPushOperator

type MetadataPushOperator struct {
	// contains filtered or unexported fields
}

func NewMetadataPushOperator

func NewMetadataPushOperator(conn connectionFetcher) *MetadataPushOperator

func (*MetadataPushOperator) Run

func (o *MetadataPushOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

type MetadataUpdater

type MetadataUpdater interface {
	UpdateTableMetadataIfNotExist(ctx context.Context, asset *pipeline.Asset) error
}

type NoMetadataUpdatedError

type NoMetadataUpdatedError struct{}

func (NoMetadataUpdatedError) Error

func (m NoMetadataUpdatedError) Error() string

type PatternCheck

type PatternCheck struct {
	// contains filtered or unexported fields
}

func (*PatternCheck) Check

func (c *PatternCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type Querier

type Querier interface {
	RunQueryWithoutResult(ctx context.Context, query *query.Query) error
	Test(ctx context.Context) error
}

type Selector

type Selector interface {
	Select(ctx context.Context, query *query.Query) ([][]interface{}, error)
	SelectWithSchema(ctx context.Context, queryObj *query.Query) (*query.QueryResult, error)
}