gbigquery

Mar 10, 2024

Changes in this version

+ var ErrMissingColumnSpec = errors.New("column spec cannot be empty")

+ var ErrMissingDataset = errors.New("dataset name cannot be empty")

+ var ErrMissingTableName = errors.New("table name cannot be empty")

+ var ErrMissingTableSpec = errors.New("table spec cannot be empty")

+ type Column struct

+ Description string

+ Fields []Column

+ Mode string

+ Name string

+ NameFromId *NameFromId

+ Type string

+ ValueFromId string

+ type DatasetCreation struct

+ Description string

+ Location string

+ type NameFromId struct

+ Prefix string

+ Preset []string

+ SuffixFromId string

+ type SinkConfig struct

+ DiscardInvalidData bool

+ Tables []Table

+ func NewSinkConfig(spec *entity.Spec) (sc SinkConfig, err error)

+ func (sc SinkConfig) Validate() error

+ type Table struct

+ Columns []Column

+ Dataset string

+ DatasetCreation *DatasetCreation

+ InsertIdFromId string

+ Name string

+ TableCreation *TableCreation

+ type TableCreation struct

+ Clustering []string

+ Description string

+ RequirePartitionFilter bool

+ TimePartitioning *TimePartitioning

+ type TimePartitioning struct

+ ExpirationHours int

+ Field string

+ Type string

Apr 20, 2023

Changes in this version

type Config

+ Client BigQueryClient

+ Creds *google.Credentials

Apr 4, 2023

Changes in this version

+ const DefaultBigQueryDatasetLocation

+ const TableUpdateBackoffTime

+ func NewBigQueryClient(c entity.Config, client *bigquery.Client) *defaultBigQueryClient

+ type BigQueryClient interface

+ Close func() error

+ CreateDataset func(ctx context.Context, id string, md *bigquery.DatasetMetadata) error

+ CreateDatasetRef func(datasetId string) *bigquery.Dataset

+ CreateTable func(ctx context.Context, datasetId string, tableId string, ...) (*bigquery.Table, error)

+ CreateTableRef func(datasetId string, tableId string) *bigquery.Table

+ GetDatasetMetadata func(ctx context.Context, dataset *bigquery.Dataset) (*bigquery.DatasetMetadata, DatasetTableStatus, error)

+ GetTableInserter func(table *bigquery.Table) BigQueryInserter

+ GetTableMetadata func(ctx context.Context, table *bigquery.Table) (*bigquery.TableMetadata, DatasetTableStatus, error)

+ UpdateTable func(ctx context.Context, table *bigquery.Table, tm bigquery.TableMetadataToUpdate, ...) (*bigquery.TableMetadata, error)

+ type BigQueryInserter interface

+ Put func(ctx context.Context, src any) error

+ type Columns map[string]entity.Column

+ type Config struct

+ ProjectId string

+ type DatasetTableStatus int

+ const Existent

+ const NonExistent

+ const Unknown

+ type LoaderFactory struct

+ func NewLoaderFactory(ctx context.Context, config Config, bqClient BigQueryClient) (*LoaderFactory, error)

+ func (lf *LoaderFactory) Close(ctx context.Context) error

+ func (lf *LoaderFactory) NewLoader(ctx context.Context, c entity.Config) (entity.Loader, error)

+ func (lf *LoaderFactory) NewSinkExtractor(ctx context.Context, c entity.Config) (entity.Extractor, error)

+ func (lf *LoaderFactory) SinkId() string

+ type Row struct

+ InsertId string

+ func NewRow() *Row

+ func (r *Row) AddItem(item *RowItem)

+ func (r *Row) Save() (map[string]bigquery.Value, string, error)

+ func (r *Row) Size() int

+ type RowItem struct

+ Name string

+ Value any