bigquery

package
v0.74.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 29, 2024 License: Apache-2.0 Imports: 12 Imported by: 0

README

Google BigQuery Service

This service is google.golang.org/api/bigquery/v2.Service proxy

To check all supported method run

     endly -s='gcp/bigquery'

To check method contract run endly -s='gcp/bigquery' -a=methodName

    endly -s='gcp/bigquery' -a='datasetsList'

References:

Usage:
Listing dataset

endly -run='gcp/bigquery:datasetsList' projectID=myProject

Query with destination table

endly -r=query authWith=myGoogleSecrets.json

@query.yaml

init:
  dataset: myDataset
  '!gcpCredentials': $params.authWith

pipeline:
  query:
    action: gcp/bigquery:query
    credentials: $gcpCredentials
    query: SELECT * FROM mySourceTable
    allowlargeresults: false
    defaultdataset:
      projectid: ${gcp.projectID}
      datasetid: $dataset
    destinationtable:
      projectid: ${gcp.projectID}
      datasetid: $dataset
      tableid: myTable
    writedisposition: WRITE_APPEND
Creating Materialize View

endly -r=mv authWith=myGoogleSecrets.json

@mv.yaml

init:
  '!gcpCredentials': $params.authWith

pipeline:
  query:
    action: gcp/bigquery:tablesInsert
    credentials:  gcpCredettials

    kind: bigquery#table
    projectId: ${gcp.projectID}
    datasetId: myDataset
    table:
      tableReference:
        projectId: ${gcp.projectID}
        datasetId: myDataset
        tableId: myTable_mv
      materializedView:
        query: SELECT SUM(columnA) AS columnA, MIN(col
Table copy

endly -r=copy authWith=myGoogleSecrets.json

@copy.yaml

init:
  '!gcpCredentials': $params.authWith
  gcpSecrets: ${secrets.$gcpCredentials}
  i: 0

  src:
    projectID: $gcpSecrets.ProjectID
    datasetID: db1
  dest:
    projectID: $gcpSecrets.ProjectID
    datasetID: db1e2e

pipeline:
  registerSource:
    action: dsunit:register
    datastore: ${src.datasetID}
    config:
      driverName: bigquery
      credentials: $gcpCredentials
      parameters:
        datasetId: $src.datasetID

  readTables:
    action: dsunit:query
    datastore: ${src.datasetID}
    SQL: SELECT table_id AS table FROM `${src.projectID}.${src.datasetID}.__TABLES__`
    post:
      dataset: $Records


  copyTables:
    loop:
      action: print
      message: $i/$Len($dataset) -> $dataset[$i].table

    copyTable:
      action: gcp/bigquery:copy
      logging: false
      credentials: $gcpCredentials
      sourceTable:
        projectID: ${src.projectID}
        datasetID: ${src.datasetID}
        tableID: $dataset[$i].table
      destinationTable:
        projectID: ${dest.projectID}
        datasetID: ${dest.datasetID}
        tableID: $dataset[$i].table

    inc:
      action: nop
      init:
        _ : $i++
    goto:
      when: $i < $Len($dataset)
      action: goto
      task: copyTables
Schema patch
  1. Patched with table template:

endly -r=patch_with_template authWith=myGoogleSecrets.json

@patch_with_template.yaml

pipeline:
  patch:
    action: gcp/bigquery:patch
    credentials: $gcpCredentials
    table: ${projectID}:bqtail.dummy_temp
    template: ${projectID}:bqtail.dummy
  1. Patched with table schema:

endly -r=patch_with_schema authWith=myGoogleSecrets.json

@patch_with_schema.yaml

pipeline:

  getTable:
    action: gcp/bigquery:table
    credentials: $gcpCredentials
    table: ${projectID}:bqtail.dummy
    dest:
      URL: /tmp/mytable.json


  patch:
    init:
      table: $Cat('/tmp/mytable.json')
      tableMap: $AsMap($table)
      schema: $tableMap.schema
    action: gcp/bigquery:patch
    credentials: $gcpCredentials
    table: ${projectID}:bqtail.dummy_temp
    schema: $schema
  1. Patch in loop;

endly -r=patch authWith=myGoogleSecret template='my_project:my_dataset.myTableTempalte' dataset=mydataset criteria='narrowing_value' '

@patch.yaml

init:
  'gcpCredentials': $params.authWith
  gcpSecrets: ${secrets.$gcpCredentials}
  projectID: $gcpSecrets.ProjectID
  '!template': $params.template
  '!dataset': $params.dataset
  '!criteria': $params.criteria
  i: 0


pipeline:
  registerSource:
    action: dsunit:register
    datastore: ${dataset}
    config:
      driverName: bigquery
      credentials: $gcpCredentials
      parameters:
        datasetId: $dataset

  readTables:
    action: dsunit:query
    datastore: $dataset
    SQL: SELECT project_id, dataset_id, table_id  FROM `${projectID}.${dataset}.__TABLES__` WHERE table_id LIKE '%${criteria}%'
    post:
      tables: $Records


  patchTable:
    loop:
      action: print
      message: Patching $i/$Len($tables) -> $tables[$i].table_id

    patch:
      action: gcp/bigquery:patch
      logging: false
      template: $template
      tableId: $tables[$i].table_id
      datasetId: $tables[$i].dataset_id
      projectId: $tables[$i].project_id
      credentials: $gcpCredentials

    inc:
      action: nop
      init:
        _: $i++
    goto:
      when: $i < $Len($tables)
      action: goto
      task: patchTable

Documentation

Index

Constants

View Source 
const (
	//ServiceID Google BigQuery Service ID.
	ServiceID = "gcp/bigquery"
)

Variables

This section is empty.

Functions

func InitRequest 

func InitRequest(context *endly.Context, rawRequest map[string]interface{}) error

func New 

func New() endly.Service

New creates a new BigQuery service.

func NewTableReference 

func NewTableReference(table string) (*bigquery.TableReference, error)

NewTableReference creates a table reference for table in the following syntax [project:]dataset.table

Types

type CopyRequest 

type CopyRequest struct {
	bigquery.JobConfigurationTableCopy
	Project string
	Job     *bigquery.JobReference
	Async   bool `description:"if set true, function does not wait for job completion"`
}

CopyRequest represents copy request

func (*CopyRequest) Validate 

func (r *CopyRequest) Validate() error

Validate checks if request is valid

type CtxClient 

type CtxClient struct {
	*gcp.AbstractClient
	// contains filtered or unexported fields
}

CtxClient represents context client

func GetClient 

func GetClient(context *endly.Context) (*CtxClient, error)

func (*CtxClient) Service 

func (s *CtxClient) Service() interface{}

func (*CtxClient) SetService 

func (s *CtxClient) SetService(service interface{}) error

type JobWaitRequest 

type JobWaitRequest struct {
	Job *bigquery.JobReference
}

func (*JobWaitRequest) Validate 

func (r *JobWaitRequest) Validate() error

Validate checks if request is valid

type LoadRequest 

type LoadRequest struct {
	bigquery.JobConfigurationLoad
	Job     *bigquery.JobReference
	Project string
	Async   bool `description:"if set true, function does not wait for job completion"`
}

LoadRequest represents load request

func (*LoadRequest) Validate 

func (r *LoadRequest) Validate() error

Validate checks if request is valid

type PatchRequest 

type PatchRequest struct {
	*bigquery.TableReference
	Table       string
	TemplateRef *bigquery.TableReference
	Template    string
	Schema      *bigquery.TableSchema
}

PatchRequest represents a patch request

func (*PatchRequest) Init 

func (r *PatchRequest) Init() (err error)

Init initialises request

func (*PatchRequest) Validate 

func (r *PatchRequest) Validate() (err error)

Validate checks if request is valid

type PatchResponse 

type PatchResponse struct {
	*bigquery.Table
}

type QueryRequest 

type QueryRequest struct {
	bigquery.JobConfigurationQuery
	Job     *bigquery.JobReference
	Project string
	Async   bool `description:"if set true, function does not wait for job completion"`
}

QueryRequest represents query request

func (*QueryRequest) Validate 

func (r *QueryRequest) Validate() error

Validate checks if request is valid

type TableRequest 

type TableRequest struct {
	Table string
	*bigquery.TableReference
	Dest *location.Resource
}

TableRequest table request

func (*TableRequest) Init 

func (r *TableRequest) Init() (err error)

func (*TableRequest) Validate 

func (r *TableRequest) Validate() (err error)

type TableResponse 

type TableResponse struct {
	Table *bigquery.Table
}

TableResponse table response

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.