bigquery

Documentation

Index

• func NewMaterializer(fullRefresh bool) *pipeline.Materializer
• type AcceptedValuesCheck
  • func (c *AcceptedValuesCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error
• type BasicOperator
  • func NewBasicOperator(conn connectionFetcher, extractor queryExtractor, materializer materializer) *BasicOperator
  • func (o BasicOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
  • func (o BasicOperator) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error
• type Client
  • func NewDB(c *Config) (*Client, error)
  • func (d *Client) GetIngestrURI() (string, error)
  • func (d *Client) IsValid(ctx context.Context, query *query.Query) (bool, error)
  • func (d *Client) RunQueryWithoutResult(ctx context.Context, query *query.Query) error
  • func (d *Client) Select(ctx context.Context, query *query.Query) ([][]interface{}, error)
• type ColumnCheckOperator
  • func NewColumnCheckOperator(manager connectionFetcher) (*ColumnCheckOperator, error)
  • func (o ColumnCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
• type Config
  • func (c Config) GetConnectionURI() (string, error)
  • func (c Config) GetIngestrURI() (string, error)
  • func (c Config) IsValid() bool
• type CustomCheck
  • func (c *CustomCheck) Check(ctx context.Context, ti *scheduler.CustomCheckInstance) error
• type CustomCheckOperator
  • func NewCustomCheckOperator(manager connectionFetcher) (*CustomCheckOperator, error)
  • func (o *CustomCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error
• type DB
• type NonNegativeCheck
  • func (c *NonNegativeCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error
• type NotNullCheck
  • func (c *NotNullCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error
• type PositiveCheck
  • func (c *PositiveCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error
• type Querier
• type Selector
• type UniqueCheck
  • func (c *UniqueCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

Functions

func NewMaterializer

func NewMaterializer(fullRefresh bool) *pipeline.Materializer

Types

type AcceptedValuesCheck

type AcceptedValuesCheck struct {
	// contains filtered or unexported fields
}

func (*AcceptedValuesCheck) Check

func (c *AcceptedValuesCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type BasicOperator

type BasicOperator struct {
	// contains filtered or unexported fields
}

func NewBasicOperator

func NewBasicOperator(conn connectionFetcher, extractor queryExtractor, materializer materializer) *BasicOperator

func (BasicOperator) Run

func (o BasicOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

func (BasicOperator) RunTask

func (o BasicOperator) RunTask(ctx context.Context, p *pipeline.Pipeline, t *pipeline.Asset) error

type Client

type Client struct {
	// contains filtered or unexported fields
}

func NewDB

func NewDB(c *Config) (*Client, error)

func (*Client) GetIngestrURI

func (d *Client) GetIngestrURI() (string, error)

func (*Client) IsValid

func (d *Client) IsValid(ctx context.Context, query *query.Query) (bool, error)

func (*Client) RunQueryWithoutResult

func (d *Client) RunQueryWithoutResult(ctx context.Context, query *query.Query) error

func (*Client) Select

func (d *Client) Select(ctx context.Context, query *query.Query) ([][]interface{}, error)

type ColumnCheckOperator

type ColumnCheckOperator struct {
	// contains filtered or unexported fields
}

func NewColumnCheckOperator

func NewColumnCheckOperator(manager connectionFetcher) (*ColumnCheckOperator, error)

func (ColumnCheckOperator) Run

func (o ColumnCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

type Config

type Config struct {
	ProjectID           string `envconfig:"BIGQUERY_PROJECT"`
	CredentialsFilePath string `envconfig:"BIGQUERY_CREDENTIALS_FILE"`
	CredentialsJSON     string
	Credentials         *google.Credentials
	Location            string `envconfig:"BIGQUERY_LOCATION"`
}

func (Config) GetConnectionURI

func (c Config) GetConnectionURI() (string, error)

func (Config) GetIngestrURI

func (c Config) GetIngestrURI() (string, error)

func (Config) IsValid

func (c Config) IsValid() bool

type CustomCheck

type CustomCheck struct {
	// contains filtered or unexported fields
}

func (*CustomCheck) Check

func (c *CustomCheck) Check(ctx context.Context, ti *scheduler.CustomCheckInstance) error

type CustomCheckOperator

type CustomCheckOperator struct {
	// contains filtered or unexported fields
}

func NewCustomCheckOperator

func NewCustomCheckOperator(manager connectionFetcher) (*CustomCheckOperator, error)

func (*CustomCheckOperator) Run

func (o *CustomCheckOperator) Run(ctx context.Context, ti scheduler.TaskInstance) error

type DB

type DB interface {
	Querier
	Selector
}

type NonNegativeCheck

type NonNegativeCheck struct {
	// contains filtered or unexported fields
}

func (*NonNegativeCheck) Check

func (c *NonNegativeCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type NotNullCheck

type NotNullCheck struct {
	// contains filtered or unexported fields
}

func (*NotNullCheck) Check

func (c *NotNullCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type PositiveCheck

type PositiveCheck struct {
	// contains filtered or unexported fields
}

func (*PositiveCheck) Check

func (c *PositiveCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error

type Querier

type Querier interface {
	RunQueryWithoutResult(ctx context.Context, query *query.Query) error
}

type Selector

type Selector interface {
	Select(ctx context.Context, query *query.Query) ([][]interface{}, error)
}

type UniqueCheck

type UniqueCheck struct {
	// contains filtered or unexported fields
}

func (*UniqueCheck) Check

func (c *UniqueCheck) Check(ctx context.Context, ti *scheduler.ColumnCheckInstance) error