xbigquery

Documentation

Index

• Constants
• func NewBigQueryClient(id string, client *bigquery.Client) *defaultBigQueryClient
• type BigQueryClient
• type BigQueryInserter
• type Columns
• type DatasetTableStatus
• type Loader
  • func NewLoader(ctx context.Context, spec *model.Spec, id string, client BigQueryClient, ...) (*Loader, error)
  • func (l *Loader) Shutdown()
  • func (l *Loader) StreamLoad(ctx context.Context, data []*model.Transformed) (string, error, bool)
• type Row
  • func NewRow() *Row
  • func (r *Row) AddItem(item *RowItem)
  • func (r *Row) Save() (map[string]bigquery.Value, string, error)
  • func (r *Row) Size() int
• type RowItem

Constants

const (
	TableUpdateBackoffTime         = 8 * time.Second
	DefaultBigQueryDatasetLocation = "EU"
)

Functions

func NewBigQueryClient

func NewBigQueryClient(id string, client *bigquery.Client) *defaultBigQueryClient

NewBigQueryClient provides a concrete wrapper client for internal usage by the Loader

Types

type BigQueryClient

type BigQueryClient interface {
	GetDatasetMetadata(ctx context.Context, dataset *bigquery.Dataset) (*bigquery.DatasetMetadata, DatasetTableStatus, error)
	CreateDatasetRef(datasetId string) *bigquery.Dataset
	CreateDataset(ctx context.Context, id string, md *bigquery.DatasetMetadata) error
	GetTableMetadata(ctx context.Context, table *bigquery.Table) (*bigquery.TableMetadata, DatasetTableStatus, error)
	CreateTableRef(datasetId string, tableId string) *bigquery.Table
	CreateTable(ctx context.Context, datasetId string, tableId string, tm *bigquery.TableMetadata) (*bigquery.Table, error)
	GetTableInserter(table *bigquery.Table) BigQueryInserter
	UpdateTable(ctx context.Context, table *bigquery.Table, tm bigquery.TableMetadataToUpdate, etag string) (*bigquery.TableMetadata, error)
}

type BigQueryInserter

type BigQueryInserter interface {
	Put(ctx context.Context, src any) error
}

type Columns

type Columns map[string]model.Column

type DatasetTableStatus

type DatasetTableStatus int

const (
	Unknown DatasetTableStatus = iota
	Existent
	NonExistent
)

type Loader

type Loader struct {
	// contains filtered or unexported fields
}

func NewLoader

func NewLoader(
	ctx context.Context,
	spec *model.Spec,
	id string,
	client BigQueryClient,
	metadataMutex *sync.Mutex) (*Loader, error)

func (*Loader) Shutdown

func (l *Loader) Shutdown()

func (*Loader) StreamLoad

func (l *Loader) StreamLoad(ctx context.Context, data []*model.Transformed) (string, error, bool)

type Row

type Row struct {
	InsertId string
	// contains filtered or unexported fields
}

func NewRow

func NewRow() *Row

func (*Row) AddItem

func (r *Row) AddItem(item *RowItem)

func (*Row) Save

func (r *Row) Save() (map[string]bigquery.Value, string, error)

Save is required for implementing the BigQuery ValueSaver interface, as used by the bigquery.Inserter

func (*Row) Size

func (r *Row) Size() int

type RowItem

type RowItem struct {
	Name  string
	Value any
}