bigquery

package
v0.7.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 11, 2023 License: Apache-2.0 Imports: 17 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ConcurrentTicketPerSec = 5
	ConcurrentLimit        = 20
)
View Source 
const (
	EntityDataset = "dataset"

	DatesetNameSections = 2
	TableNameSections   = 3
)
View Source 
const (
	ModeNullable = "nullable"
	ModeRequired = "required"
	ModeRepeated = "repeated"

	EntityResourceSchema = "bigquery_schema"
)
View Source 
const (
	KindDataset       string = "dataset"
	KindTable         string = "table"
	KindView          string = "view"
	KindExternalTable string = "external_table"
)
View Source 
const (
	EntityExternalTable = "resource_external_table"
)
View Source 
const (
	EntityTable = "resource_table"
)
View Source 
const (
	EntityView = "resource_view"
)

Variables

This section is empty.

Functions

func BackupResources 

func BackupResources(ctx context.Context, backup *resource.Backup, resources []*resource.Resource, client Client) (*resource.BackupResult, error)

func BackupTable 

func BackupTable(ctx context.Context, backup *resource.Backup, source *resource.Resource, client Client) (string, error)

func BatchesFrom 

func BatchesFrom(resources []*resource.Resource, provider ClientProvider) (map[string]*Batch, error)

func ConfigAs 

func ConfigAs[T any](mapping map[string]any, key string) T

func ConvertSpecTo 

func ConvertSpecTo[T DatasetDetails | Table | View | ExternalTable](res *resource.Resource) (*T, error)

func CopyTable 

func CopyTable(ctx context.Context, source, destination TableResourceHandle) error

func CreateIfDatasetDoesNotExist 

func CreateIfDatasetDoesNotExist(ctx context.Context, client Client, dataset Dataset) error

func DestinationExpiry 

func DestinationExpiry(backup *resource.Backup) (time.Time, error)

func DestinationName 

func DestinationName(sourceDatasetName, sourceName string, backup *resource.Backup) string

func ResourceNameFor 

func ResourceNameFor(res *resource.Resource) (string, error)

func URNFor 

func URNFor(res *resource.Resource) (string, error)

func ValidateName 

func ValidateName(res *resource.Resource) error

Types

type BQCopier 

type BQCopier interface {
	Run(context.Context) (*bigquery.Job, error)
}

type BQJob 

type BQJob interface {
	Wait(context.Context) (*bigquery.JobStatus, error)
}

type Batch 

type Batch struct {
	Dataset        Dataset
	DatasetDetails *resource.Resource

	Tables         []*resource.Resource
	ExternalTables []*resource.Resource
	Views          []*resource.Resource
	// contains filtered or unexported fields
}

func (*Batch) DatasetOrDefault 

func (b *Batch) DatasetOrDefault() (*resource.Resource, error)

func (*Batch) QueueJobs 

func (b *Batch) QueueJobs(ctx context.Context, account string, runner *parallel.Runner) error

type BqClient 

type BqClient struct {
	// contains filtered or unexported fields
}

func NewClient 

func NewClient(ctx context.Context, svcAccount string) (*BqClient, error)

func (*BqClient) Close 

func (c *BqClient) Close()

func (*BqClient) DatasetHandleFrom 

func (c *BqClient) DatasetHandleFrom(ds Dataset) ResourceHandle

func (*BqClient) ExternalTableHandleFrom 

func (c *BqClient) ExternalTableHandleFrom(ds Dataset, name string) ResourceHandle

func (*BqClient) TableHandleFrom 

func (c *BqClient) TableHandleFrom(ds Dataset, name string) TableResourceHandle

func (*BqClient) ViewHandleFrom 

func (c *BqClient) ViewHandleFrom(ds Dataset, name string) ResourceHandle

type BqClientProvider 

type BqClientProvider struct{}

func NewClientProvider 

func NewClientProvider() *BqClientProvider

func (BqClientProvider) Get 

func (BqClientProvider) Get(ctx context.Context, account string) (Client, error)

type BqDataset 

type BqDataset interface {
	Create(context.Context, *bigquery.DatasetMetadata) error
	Update(context.Context, bigquery.DatasetMetadataToUpdate, string) (*bigquery.DatasetMetadata, error)
	Metadata(context.Context) (*bigquery.DatasetMetadata, error)
}

type BqTable 

type BqTable interface {
	Create(context.Context, *bigquery.TableMetadata) error
	Update(context.Context, bigquery.TableMetadataToUpdate, string, ...bigquery.TableUpdateOption) (*bigquery.TableMetadata, error)
	Metadata(ctx context.Context, opts ...bigquery.TableMetadataOption) (*bigquery.TableMetadata, error)
	CopierFrom(srcs ...*bigquery.Table) *bigquery.Copier
}

type Client 

type Client interface {
	DatasetHandleFrom(dataset Dataset) ResourceHandle
	TableHandleFrom(dataset Dataset, name string) TableResourceHandle
	ExternalTableHandleFrom(dataset Dataset, name string) ResourceHandle
	ViewHandleFrom(dataset Dataset, name string) ResourceHandle
	Close()
}

type ClientProvider 

type ClientProvider interface {
	Get(ctx context.Context, account string) (Client, error)
}

type Cluster 

type Cluster struct {
	Using []string `mapstructure:"using,omitempty"`
}

func (Cluster) Validate 

func (c Cluster) Validate() error

type Copier 

type Copier struct {
	// contains filtered or unexported fields
}

func NewCopier 

func NewCopier(bqCopier BQCopier) *Copier

func (Copier) Run 

func (c Copier) Run(ctx context.Context) (CopyJob, error)

type CopyJob 

type CopyJob interface {
	Wait(ctx context.Context) error
}

type Dataset 

type Dataset struct {
	Project     string
	DatasetName string
}

func DataSetFor 

func DataSetFor(res *resource.Resource) (Dataset, error)

func DataSetFrom 

func DataSetFrom(project, datasetName string) (Dataset, error)

func DestinationDataset 

func DestinationDataset(project string, backup *resource.Backup) (Dataset, error)

func (Dataset) FullName 

func (d Dataset) FullName() string

type DatasetDetails 

type DatasetDetails struct {
	Name resource.Name

	Description string                 `mapstructure:"description,omitempty"`
	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (DatasetDetails) FullName 

func (d DatasetDetails) FullName() string

func (DatasetDetails) Validate 

func (DatasetDetails) Validate() error

type DatasetHandle 

type DatasetHandle struct {
	// contains filtered or unexported fields
}

func NewDatasetHandle 

func NewDatasetHandle(ds BqDataset) *DatasetHandle

func (DatasetHandle) Create 

func (d DatasetHandle) Create(ctx context.Context, res *resource.Resource) error

func (DatasetHandle) Exists 

func (d DatasetHandle) Exists(ctx context.Context) bool

func (DatasetHandle) Update 

func (d DatasetHandle) Update(ctx context.Context, res *resource.Resource) error

type ExternalSource 

type ExternalSource struct {
	SourceType string   `mapstructure:"type,omitempty"`
	SourceURIs []string `mapstructure:"uris,omitempty"`

	// Additional configs for CSV, GoogleSheets, Bigtable, and Parquet formats.
	Config map[string]interface{} `mapstructure:"config"`
}

func (ExternalSource) Validate 

func (e ExternalSource) Validate() error

type ExternalTable 

type ExternalTable struct {
	Name resource.Name

	Description string          `mapstructure:"description,omitempty"`
	Schema      Schema          `mapstructure:"schema,omitempty"`
	Source      *ExternalSource `mapstructure:"source,omitempty"`

	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (*ExternalTable) FullName 

func (e *ExternalTable) FullName() string

func (*ExternalTable) Validate 

func (e *ExternalTable) Validate() error

type ExternalTableHandle 

type ExternalTableHandle struct {
	// contains filtered or unexported fields
}

func NewExternalTableHandle 

func NewExternalTableHandle(bq BqTable) *ExternalTableHandle

func (ExternalTableHandle) Create 

func (et ExternalTableHandle) Create(ctx context.Context, res *resource.Resource) error

func (ExternalTableHandle) Exists 

func (et ExternalTableHandle) Exists(ctx context.Context) bool

func (ExternalTableHandle) Update 

func (et ExternalTableHandle) Update(ctx context.Context, res *resource.Resource) error

type Field 

type Field struct {
	Name        string `mapstructure:"name,omitempty"`
	Type        string `mapstructure:"type,omitempty"`
	Description string `mapstructure:"description,omitempty"`
	Mode        string `mapstructure:"mode,omitempty"`

	// optional sub-schema, when record type
	Schema Schema `mapstructure:"schema,omitempty"`
}

func (Field) Validate 

func (f Field) Validate() error

type JobHandle 

type JobHandle struct {
	// contains filtered or unexported fields
}

func NewJob 

func NewJob(job BQJob) *JobHandle

func (JobHandle) Wait 

func (j JobHandle) Wait(ctx context.Context) error

type Partition 

type Partition struct {
	Field string `mapstructure:"field,omitempty"`

	Type       string `mapstructure:"type,omitempty"`
	Expiration int64  `mapstructure:"expiration,omitempty"`

	Range *Range `mapstructure:"range,omitempty"`
}

func (Partition) Validate 

func (p Partition) Validate() error

type Range 

type Range struct {
	Start    int64 `mapstructure:"start,omitempty"`
	End      int64 `mapstructure:"end,omitempty"`
	Interval int64 `mapstructure:"interval,omitempty"`
}

type ResourceHandle 

type ResourceHandle interface {
	Create(ctx context.Context, res *resource.Resource) error
	Update(ctx context.Context, res *resource.Resource) error
	Exists(ctx context.Context) bool
}

type Schema 

type Schema []Field

func (Schema) Validate 

func (s Schema) Validate() error

type SecretProvider 

type SecretProvider interface {
	GetSecret(ctx context.Context, tnnt tenant.Tenant, key string) (*tenant.PlainTextSecret, error)
}

type Store 

type Store struct {
	// contains filtered or unexported fields
}

func NewBigqueryDataStore 

func NewBigqueryDataStore(secretProvider SecretProvider, clientProvider ClientProvider) *Store

func (Store) Backup 

func (s Store) Backup(ctx context.Context, backup *resource.Backup, resources []*resource.Resource) (*resource.BackupResult, error)

func (Store) BatchUpdate 

func (s Store) BatchUpdate(ctx context.Context, resources []*resource.Resource) error

func (Store) Create 

func (s Store) Create(ctx context.Context, res *resource.Resource) error

func (Store) GetURN 

func (Store) GetURN(res *resource.Resource) (string, error)

func (Store) Update 

func (s Store) Update(ctx context.Context, res *resource.Resource) error

func (Store) Validate 

func (Store) Validate(r *resource.Resource) error

type Table 

type Table struct {
	Name resource.Name

	Description string     `mapstructure:"description,omitempty"`
	Schema      Schema     `mapstructure:"schema,omitempty"`
	Cluster     *Cluster   `mapstructure:"cluster,omitempty"`
	Partition   *Partition `mapstructure:"partition,omitempty"`

	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (*Table) FullName 

func (t *Table) FullName() string

func (*Table) Validate 

func (t *Table) Validate() error

type TableCopier 

type TableCopier interface {
	Run(ctx context.Context) (CopyJob, error)
}

type TableHandle 

type TableHandle struct {
	// contains filtered or unexported fields
}

func NewTableHandle 

func NewTableHandle(bq BqTable) *TableHandle

func (TableHandle) CopierFrom 

func (t TableHandle) CopierFrom(source TableResourceHandle) (TableCopier, error)

func (TableHandle) Create 

func (t TableHandle) Create(ctx context.Context, res *resource.Resource) error

func (TableHandle) Exists 

func (t TableHandle) Exists(ctx context.Context) bool

func (TableHandle) GetBQTable 

func (t TableHandle) GetBQTable() (*bigquery.Table, error)

func (TableHandle) Update 

func (t TableHandle) Update(ctx context.Context, res *resource.Resource) error

func (TableHandle) UpdateExpiry 

func (t TableHandle) UpdateExpiry(ctx context.Context, name string, expiry time.Time) error

type TableResourceHandle 

type TableResourceHandle interface {
	ResourceHandle
	GetBQTable() (*bq.Table, error)
	CopierFrom(source TableResourceHandle) (TableCopier, error)
	UpdateExpiry(ctx context.Context, name string, expiry time.Time) error
}

type View 

type View struct {
	Name resource.Name

	Description string `mapstructure:"description,omitempty"`
	ViewQuery   string `mapstructure:"view_query,omitempty"`

	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (*View) Validate 

func (v *View) Validate() error

type ViewHandle 

type ViewHandle struct {
	// contains filtered or unexported fields
}

func NewViewHandle 

func NewViewHandle(bq BqTable) *ViewHandle

func (ViewHandle) Create 

func (v ViewHandle) Create(ctx context.Context, res *resource.Resource) error

func (ViewHandle) Exists 

func (v ViewHandle) Exists(ctx context.Context) bool

func (ViewHandle) Update 

func (v ViewHandle) Update(ctx context.Context, res *resource.Resource) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.