bigquery

package
v0.17.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 23, 2024 License: Apache-2.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ConcurrentTicketPerSec = 5
	ConcurrentLimit        = 20
)
View Source 
const (
	EntityDataset = "dataset"

	DatesetNameSections = 2
	TableNameSections   = 3
)
View Source 
const (
	ModeNullable = "nullable"
	ModeRequired = "required"
	ModeRepeated = "repeated"

	EntityResourceSchema = "bigquery_schema"
)
View Source 
const (
	KindDataset       string = "dataset"
	KindTable         string = "table"
	KindView          string = "view"
	KindExternalTable string = "external_table"
	KindRoutine       string = "routine"
	KindModel         string = "model"
)
View Source 
const (
	EntityExternalTable = "resource_external_table"
)
View Source 
const EntityModel = "resource_model"
View Source 
const EntityRoutine = "resource_routine"
View Source 
const (
	EntityTable = "resource_table"
)
View Source 
const (
	EntityView = "resource_view"
)

Variables

This section is empty.

Functions

func BackupResources 

func BackupResources(ctx context.Context, backup *resource.Backup, resources []*resource.Resource, client Client) (*resource.BackupResult, error)

func BackupTable 

func BackupTable(ctx context.Context, backup *resource.Backup, source *resource.Resource, client Client) (string, error)

func BatchesFrom 

func BatchesFrom(resources []*resource.Resource, provider ClientProvider) (map[string]*Batch, error)

func ConfigAs 

func ConfigAs[T any](mapping map[string]any, key string) T

func ConvertSpecTo 

func ConvertSpecTo[T DatasetDetails | Table | View | ExternalTable](res *resource.Resource) (*T, error)

func CopyTable 

func CopyTable(ctx context.Context, source, destination TableResourceHandle) error

func CreateIfDatasetDoesNotExist 

func CreateIfDatasetDoesNotExist(ctx context.Context, client Client, dataset Dataset) error

func DestinationExpiry 

func DestinationExpiry(backup *resource.Backup) (time.Time, error)

func DestinationName 

func DestinationName(sourceDatasetName, sourceName string, backup *resource.Backup) string

func ResourceNameFor 

func ResourceNameFor(name resource.Name, kind string) (string, error)

func URNFor 

func URNFor(res *resource.Resource) (resource.URN, error)

func ValidateName 

func ValidateName(res *resource.Resource) error

Types

type BQCopier 

type BQCopier interface {
	Run(context.Context) (*bigquery.Job, error)
}

type BQJob 

type BQJob interface {
	Wait(context.Context) (*bigquery.JobStatus, error)
}

type Batch 

type Batch struct {
	Dataset        Dataset
	DatasetDetails *resource.Resource

	Tables         []*resource.Resource
	ExternalTables []*resource.Resource
	Views          []*resource.Resource
	// contains filtered or unexported fields
}

func (*Batch) QueueJobs 

func (b *Batch) QueueJobs(ctx context.Context, account string, runner *parallel.Runner) error

type BqClient 

type BqClient struct {
	*bigquery.Client
}

func NewClient 

func NewClient(ctx context.Context, svcAccount string) (*BqClient, error)

func (*BqClient) BulkGetDDLView added in v0.10.1 

func (c *BqClient) BulkGetDDLView(ctx context.Context, pd ProjectDataset, names []string) (map[ResourceURN]string, error)

func (*BqClient) DatasetHandleFrom 

func (c *BqClient) DatasetHandleFrom(ds Dataset) ResourceHandle

func (*BqClient) ExternalTableHandleFrom 

func (c *BqClient) ExternalTableHandleFrom(ds Dataset, name string) ResourceHandle

func (*BqClient) ModelHandleFrom added in v0.16.0 

func (c *BqClient) ModelHandleFrom(ds Dataset, name string) ResourceHandle

func (*BqClient) RoutineHandleFrom added in v0.13.1 

func (c *BqClient) RoutineHandleFrom(ds Dataset, name string) ResourceHandle

func (*BqClient) TableHandleFrom 

func (c *BqClient) TableHandleFrom(ds Dataset, name string) TableResourceHandle

func (*BqClient) ViewHandleFrom 

func (c *BqClient) ViewHandleFrom(ds Dataset, name string) ResourceHandle

type BqClientProvider 

type BqClientProvider struct{}

func NewClientProvider 

func NewClientProvider() *BqClientProvider

func (BqClientProvider) Get 

func (BqClientProvider) Get(ctx context.Context, account string) (Client, error)

type BqDataset 

type BqDataset interface {
	Create(context.Context, *bigquery.DatasetMetadata) error
	Update(context.Context, bigquery.DatasetMetadataToUpdate, string) (*bigquery.DatasetMetadata, error)
	Metadata(context.Context) (*bigquery.DatasetMetadata, error)
}

type BqModel added in v0.16.0 

type BqModel interface {
	Metadata(ctx context.Context) (mm *bigquery.ModelMetadata, err error)
}

BqModel is BigQuery Model

type BqRoutine added in v0.13.1 

type BqRoutine interface {
	Create(ctx context.Context, rm *bigquery.RoutineMetadata) (err error)
	Update(ctx context.Context, upd *bigquery.RoutineMetadataToUpdate, etag string) (rm *bigquery.RoutineMetadata, err error)
	Metadata(ctx context.Context) (*bigquery.RoutineMetadata, error)
}

BqRoutine is including UDF, Store Procedure, Table Function

type BqTable 

type BqTable interface {
	Create(context.Context, *bigquery.TableMetadata) error
	Update(context.Context, bigquery.TableMetadataToUpdate, string, ...bigquery.TableUpdateOption) (*bigquery.TableMetadata, error)
	Metadata(ctx context.Context, opts ...bigquery.TableMetadataOption) (*bigquery.TableMetadata, error)
	CopierFrom(srcs ...*bigquery.Table) *bigquery.Copier
}

type Client 

type Client interface {
	DatasetHandleFrom(dataset Dataset) ResourceHandle
	TableHandleFrom(dataset Dataset, name string) TableResourceHandle
	ExternalTableHandleFrom(dataset Dataset, name string) ResourceHandle
	ViewHandleFrom(dataset Dataset, name string) ResourceHandle
	RoutineHandleFrom(ds Dataset, name string) ResourceHandle
	ModelHandleFrom(ds Dataset, name string) ResourceHandle
	BulkGetDDLView(ctx context.Context, dataset ProjectDataset, names []string) (map[ResourceURN]string, error)
	Close() error
}

type ClientProvider 

type ClientProvider interface {
	Get(ctx context.Context, account string) (Client, error)
}

type Cluster 

type Cluster struct {
	Using []string `mapstructure:"using,omitempty"`
}

func (Cluster) Validate 

func (c Cluster) Validate() error

type Copier 

type Copier struct {
	// contains filtered or unexported fields
}

func NewCopier 

func NewCopier(bqCopier BQCopier) *Copier

func (Copier) Run 

func (c Copier) Run(ctx context.Context) (CopyJob, error)

type CopyJob 

type CopyJob interface {
	Wait(ctx context.Context) error
}

type Dataset 

type Dataset struct {
	Project     string
	DatasetName string
}

func DataSetFor 

func DataSetFor(name resource.Name) (Dataset, error)

func DataSetFrom 

func DataSetFrom(project, datasetName string) (Dataset, error)

func DestinationDataset 

func DestinationDataset(project string, backup *resource.Backup) (Dataset, error)

func (Dataset) FullName 

func (d Dataset) FullName() string

type DatasetDetails 

type DatasetDetails struct {
	Name resource.Name

	Description string                 `mapstructure:"description,omitempty"`
	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (DatasetDetails) FullName 

func (d DatasetDetails) FullName() string

func (DatasetDetails) Validate 

func (DatasetDetails) Validate() error

type DatasetHandle 

type DatasetHandle struct {
	// contains filtered or unexported fields
}

func NewDatasetHandle 

func NewDatasetHandle(ds BqDataset) *DatasetHandle

func (DatasetHandle) Create 

func (d DatasetHandle) Create(ctx context.Context, res *resource.Resource) error

func (DatasetHandle) Exists 

func (d DatasetHandle) Exists(ctx context.Context) bool

func (DatasetHandle) Update 

func (d DatasetHandle) Update(ctx context.Context, res *resource.Resource) error

type ExternalSource 

type ExternalSource struct {
	SourceType string   `mapstructure:"type,omitempty"`
	SourceURIs []string `mapstructure:"uris,omitempty"`

	// Additional configs for CSV, GoogleSheets, Bigtable, and Parquet formats.
	Config map[string]interface{} `mapstructure:"config"`
}

func (ExternalSource) Validate 

func (e ExternalSource) Validate() error

type ExternalTable 

type ExternalTable struct {
	Name resource.Name

	Description string          `mapstructure:"description,omitempty"`
	Schema      Schema          `mapstructure:"schema,omitempty"`
	Source      *ExternalSource `mapstructure:"source,omitempty"`

	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (*ExternalTable) FullName 

func (e *ExternalTable) FullName() string

func (*ExternalTable) Validate 

func (e *ExternalTable) Validate() error

type ExternalTableHandle 

type ExternalTableHandle struct {
	// contains filtered or unexported fields
}

func NewExternalTableHandle 

func NewExternalTableHandle(bq BqTable) *ExternalTableHandle

func (ExternalTableHandle) Create 

func (et ExternalTableHandle) Create(ctx context.Context, res *resource.Resource) error

func (ExternalTableHandle) Exists 

func (et ExternalTableHandle) Exists(ctx context.Context) bool

func (ExternalTableHandle) Update 

func (et ExternalTableHandle) Update(ctx context.Context, res *resource.Resource) error

type Field 

type Field struct {
	Name        string `mapstructure:"name,omitempty"`
	Type        string `mapstructure:"type,omitempty"`
	Description string `mapstructure:"description,omitempty"`
	Mode        string `mapstructure:"mode,omitempty"`

	// optional sub-schema, when record type
	Schema Schema `mapstructure:"schema,omitempty"`
}

func (Field) Validate 

func (f Field) Validate() error

type JobHandle 

type JobHandle struct {
	// contains filtered or unexported fields
}

func NewJob 

func NewJob(job BQJob) *JobHandle

func (JobHandle) Wait 

func (j JobHandle) Wait(ctx context.Context) error

type ModelHandle added in v0.16.0 

type ModelHandle struct {
	// contains filtered or unexported fields
}

func NewModelHandle added in v0.16.0 

func NewModelHandle(bq BqModel) *ModelHandle

func (ModelHandle) Create added in v0.16.0 

func (ModelHandle) Create(_ context.Context, _ *resource.Resource) error

func (ModelHandle) Exists added in v0.16.0 

func (r ModelHandle) Exists(ctx context.Context) bool

func (ModelHandle) Update added in v0.16.0 

func (ModelHandle) Update(_ context.Context, _ *resource.Resource) error

type Partition 

type Partition struct {
	Field string `mapstructure:"field,omitempty"`

	Type       string `mapstructure:"type,omitempty"`
	Expiration int64  `mapstructure:"expiration,omitempty"`

	Range *Range `mapstructure:"range,omitempty"`
}

func (Partition) Validate 

func (p Partition) Validate() error

type ProjectDataset added in v0.10.1 

type ProjectDataset struct {
	Project string
	Dataset string
}

type Range 

type Range struct {
	Start    int64 `mapstructure:"start,omitempty"`
	End      int64 `mapstructure:"end,omitempty"`
	Interval int64 `mapstructure:"interval,omitempty"`
}

type ResourceHandle 

type ResourceHandle interface {
	Create(ctx context.Context, res *resource.Resource) error
	Update(ctx context.Context, res *resource.Resource) error
	Exists(ctx context.Context) bool
}

type ResourceURN added in v0.10.1 

type ResourceURN struct {
	// contains filtered or unexported fields
}

func NewResourceURN added in v0.10.1 

func NewResourceURN(project, dataset, name string) (ResourceURN, error)

func NewResourceURNFromString added in v0.10.1 

func NewResourceURNFromString(urn string) (ResourceURN, error)

func (ResourceURN) Dataset added in v0.10.1 

func (n ResourceURN) Dataset() string

func (ResourceURN) Name added in v0.10.1 

func (n ResourceURN) Name() string

func (ResourceURN) Project added in v0.10.1 

func (n ResourceURN) Project() string

func (ResourceURN) URN added in v0.10.1 

func (n ResourceURN) URN() string

type ResourceURNWithUpstreams added in v0.10.2 

type ResourceURNWithUpstreams struct {
	ResourceURN ResourceURN
	Upstreams   []*ResourceURNWithUpstreams
}

type ResourceURNWithUpstreamsList added in v0.10.2 

type ResourceURNWithUpstreamsList []*ResourceURNWithUpstreams

func (ResourceURNWithUpstreamsList) FlattenUnique added in v0.10.2 

func (rs ResourceURNWithUpstreamsList) FlattenUnique() []*ResourceURNWithUpstreams

type ResourceURNs added in v0.10.1 

type ResourceURNs []ResourceURN

func (ResourceURNs) GroupByProjectDataset added in v0.10.1 

func (n ResourceURNs) GroupByProjectDataset() map[ProjectDataset][]string

type RoutineHandle added in v0.13.1 

type RoutineHandle struct {
	// contains filtered or unexported fields
}

func NewRoutineHandle added in v0.13.1 

func NewRoutineHandle(bq BqRoutine) *RoutineHandle

func (RoutineHandle) Create added in v0.13.1 

func (RoutineHandle) Create(_ context.Context, _ *resource.Resource) error

func (RoutineHandle) Exists added in v0.13.1 

func (r RoutineHandle) Exists(ctx context.Context) bool

func (RoutineHandle) Update added in v0.13.1 

func (RoutineHandle) Update(_ context.Context, _ *resource.Resource) error

type Schema 

type Schema []Field

func (Schema) Validate 

func (s Schema) Validate() error

type SecretProvider 

type SecretProvider interface {
	GetSecret(ctx context.Context, tnnt tenant.Tenant, key string) (*tenant.PlainTextSecret, error)
}

type Store 

type Store struct {
	// contains filtered or unexported fields
}

func NewBigqueryDataStore 

func NewBigqueryDataStore(secretProvider SecretProvider, clientProvider ClientProvider) *Store

func (Store) Backup 

func (s Store) Backup(ctx context.Context, backup *resource.Backup, resources []*resource.Resource) (*resource.BackupResult, error)

func (Store) BatchUpdate 

func (s Store) BatchUpdate(ctx context.Context, resources []*resource.Resource) error

func (Store) Create 

func (s Store) Create(ctx context.Context, res *resource.Resource) error

func (Store) Exist added in v0.12.0 

func (s Store) Exist(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error)

func (Store) GetURN 

func (Store) GetURN(res *resource.Resource) (resource.URN, error)

func (Store) Update 

func (s Store) Update(ctx context.Context, res *resource.Resource) error

func (Store) Validate 

func (Store) Validate(r *resource.Resource) error

type Table 

type Table struct {
	Name resource.Name

	Description string     `mapstructure:"description,omitempty"`
	Schema      Schema     `mapstructure:"schema,omitempty"`
	Cluster     *Cluster   `mapstructure:"cluster,omitempty"`
	Partition   *Partition `mapstructure:"partition,omitempty"`

	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (*Table) FullName 

func (t *Table) FullName() string

func (*Table) Validate 

func (t *Table) Validate() error

type TableCopier 

type TableCopier interface {
	Run(ctx context.Context) (CopyJob, error)
}

type TableHandle 

type TableHandle struct {
	// contains filtered or unexported fields
}

func NewTableHandle 

func NewTableHandle(bq BqTable) *TableHandle

func (TableHandle) CopierFrom 

func (t TableHandle) CopierFrom(source TableResourceHandle) (TableCopier, error)

func (TableHandle) Create 

func (t TableHandle) Create(ctx context.Context, res *resource.Resource) error

func (TableHandle) Exists 

func (t TableHandle) Exists(ctx context.Context) bool

func (TableHandle) GetBQTable 

func (t TableHandle) GetBQTable() (*bigquery.Table, error)

func (TableHandle) Update 

func (t TableHandle) Update(ctx context.Context, res *resource.Resource) error

func (TableHandle) UpdateExpiry 

func (t TableHandle) UpdateExpiry(ctx context.Context, name string, expiry time.Time) error

type TableResourceHandle 

type TableResourceHandle interface {
	ResourceHandle
	GetBQTable() (*bq.Table, error)
	CopierFrom(source TableResourceHandle) (TableCopier, error)
	UpdateExpiry(ctx context.Context, name string, expiry time.Time) error
}

type URNComponent added in v0.17.0 

type URNComponent struct {
	Project string `mapstructure:"project"`
	Dataset string `mapstructure:"dataset"`
	Name    string `mapstructure:"name"`
}

type View 

type View struct {
	Name resource.Name

	Description string `mapstructure:"description,omitempty"`
	ViewQuery   string `mapstructure:"view_query,omitempty"`

	ExtraConfig map[string]interface{} `mapstructure:",remain"`
}

func (*View) Validate 

func (v *View) Validate() error

type ViewHandle 

type ViewHandle struct {
	// contains filtered or unexported fields
}

func NewViewHandle 

func NewViewHandle(bq BqTable) *ViewHandle

func (ViewHandle) Create 

func (v ViewHandle) Create(ctx context.Context, res *resource.Resource) error

func (ViewHandle) Exists 

func (v ViewHandle) Exists(ctx context.Context) bool

func (ViewHandle) Update 

func (v ViewHandle) Update(ctx context.Context, res *resource.Resource) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.