gbigquery

Documentation

Index

• Constants
• func NewBigQueryClient(c entity.Config, client *bigquery.Client) *defaultBigQueryClient
• type BigQueryClient
• type BigQueryInserter
• type Columns
• type Config
• type DatasetTableStatus
• type LoaderFactory
  • func NewLoaderFactory(ctx context.Context, config Config, client BigQueryClient) (*LoaderFactory, error)
  • func (lf *LoaderFactory) Close(ctx context.Context) error
  • func (lf *LoaderFactory) NewLoader(ctx context.Context, c entity.Config) (entity.Loader, error)
  • func (lf *LoaderFactory) NewSinkExtractor(ctx context.Context, c entity.Config) (entity.Extractor, error)
  • func (lf *LoaderFactory) SinkId() string
• type Row
  • func NewRow() *Row
  • func (r *Row) AddItem(item *RowItem)
  • func (r *Row) Save() (map[string]bigquery.Value, string, error)
  • func (r *Row) Size() int
• type RowItem

Constants

const (
	TableUpdateBackoffTime         = 8 * time.Second
	DefaultBigQueryDatasetLocation = "EU"
)

Functions

func NewBigQueryClient

func NewBigQueryClient(c entity.Config, client *bigquery.Client) *defaultBigQueryClient

NewBigQueryClient provides a concrete wrapper client for internal usage by the Loader

Types

type BigQueryClient

type BigQueryClient interface {
	GetDatasetMetadata(ctx context.Context, dataset *bigquery.Dataset) (*bigquery.DatasetMetadata, DatasetTableStatus, error)
	CreateDatasetRef(datasetId string) *bigquery.Dataset
	CreateDataset(ctx context.Context, id string, md *bigquery.DatasetMetadata) error
	GetTableMetadata(ctx context.Context, table *bigquery.Table) (*bigquery.TableMetadata, DatasetTableStatus, error)
	CreateTableRef(datasetId string, tableId string) *bigquery.Table
	CreateTable(ctx context.Context, datasetId string, tableId string, tm *bigquery.TableMetadata) (*bigquery.Table, error)
	GetTableInserter(table *bigquery.Table) BigQueryInserter
	UpdateTable(ctx context.Context, table *bigquery.Table, tm bigquery.TableMetadataToUpdate, etag string) (*bigquery.TableMetadata, error)
	Close() error
}

type BigQueryInserter

type BigQueryInserter interface {
	Put(ctx context.Context, src any) error
}

type Columns

type Columns map[string]entity.Column

type Config

type Config struct {

	// ProjectId (required) specifies GCP project ID for this deployment.
	ProjectId string
}

type DatasetTableStatus

type DatasetTableStatus int

const (
	Unknown DatasetTableStatus = iota
	Existent
	NonExistent
)

type LoaderFactory

type LoaderFactory struct {
	// contains filtered or unexported fields
}

func NewLoaderFactory

func NewLoaderFactory(ctx context.Context, config Config, client BigQueryClient) (*LoaderFactory, error)

NewLoaderFactory creates a new BigQuery loader connector. For standard usage, set bqClient to nil, making a default BigQuery client to be created internally.

func (*LoaderFactory) Close

func (lf *LoaderFactory) Close(ctx context.Context) error

func (*LoaderFactory) NewLoader

func (lf *LoaderFactory) NewLoader(ctx context.Context, c entity.Config) (entity.Loader, error)

func (*LoaderFactory) NewSinkExtractor

func (lf *LoaderFactory) NewSinkExtractor(ctx context.Context, c entity.Config) (entity.Extractor, error)

func (*LoaderFactory) SinkId

func (lf *LoaderFactory) SinkId() string

type Row

type Row struct {
	InsertId string
	// contains filtered or unexported fields
}

func NewRow

func NewRow() *Row

func (*Row) AddItem

func (r *Row) AddItem(item *RowItem)

func (*Row) Save

func (r *Row) Save() (map[string]bigquery.Value, string, error)

Save is required for implementing the BigQuery ValueSaver interface, as used by the bigquery.Inserter

func (*Row) Size

func (r *Row) Size() int

type RowItem

type RowItem struct {
	Name  string
	Value any
}