bq2gcs

Documentation

Index

• Constants
• func CheckExpireForDateShardingTable(tableID string, expireDay int) (bool, error)
• type APIOptions
  • func WithDryRun() APIOptions
  • func WithStreamLogFn(f func(msg string)) APIOptions
  • func WithWait() APIOptions
• type DateShardingTableTarget
  • func (t *DateShardingTableTarget) Match(tableID string) (bool, error)
• type ExportHandler
  • func (h *ExportHandler) Serve(ctx context.Context, w http.ResponseWriter, r *http.Request) *handlers.HTTPResponse
• type ExportShardingTablesLimit
• type ExportsReq
• type GCSReferenceForExportShardingTables
• type GCSReferenceForExportShardingTablesReq
• type Service
  • func NewService(ctx context.Context, bq *bigquery.Client) (*Service, error)
  • func (s *Service) ExportShardingTable(ctx context.Context, to *GCSReferenceForExportShardingTables, projectID string, ...) (*bigquery.Job, error)
  • func (s *Service) ExportShardingTables(ctx context.Context, to *GCSReferenceForExportShardingTables, projectID string, ...) ([]string, error)
• type TargetTable

Constants

const (
	DefaultExportMaxTableSize  = 50 * 1024 * 1024 * 1024 * 1024
	DefaultExportMaxTableCount = 100000
)

Functions

func CheckExpireForDateShardingTable

func CheckExpireForDateShardingTable(tableID string, expireDay int) (bool, error)

CheckExpireForDateShardingTable is 指定したtableがexpireしているかを返す

Types

type APIOptions

type APIOptions func(options *apiOptions)

func WithDryRun

func WithDryRun() APIOptions

WithDryRun is 変更が発生するbigqueryのAPIは実行しない

func WithStreamLogFn

func WithStreamLogFn(f func(msg string)) APIOptions

WithStreamLogFn is Query結果を元にAPIを実行した時にログを処理できる関数を指定できる

func WithWait

func WithWait() APIOptions

WithWait is Jobが一つずつ完了するのを待ってから、次のJobを投入する

type DateShardingTableTarget

type DateShardingTableTarget struct {
	// Prefix is TableID Prefix
	Prefix string

	// ExpirationDay is 期限切れ日数
	ExpirationDay int
}

func (*DateShardingTableTarget) Match

func (t *DateShardingTableTarget) Match(tableID string) (bool, error)

type ExportHandler

type ExportHandler struct {
}

func (*ExportHandler) Serve

func (h *ExportHandler) Serve(ctx context.Context, w http.ResponseWriter, r *http.Request) *handlers.HTTPResponse

type ExportShardingTablesLimit

type ExportShardingTablesLimit struct {
	TableSize  int64
	TableCount int64
}

type ExportsReq

type ExportsReq struct {
	Project string                                  `json:"project"`
	Target  *TargetTable                            `json:"target"`
	ToGCS   *GCSReferenceForExportShardingTablesReq `json:"toGCS"`
	Limit   *ExportShardingTablesLimit              `json:"limit"`
}

type GCSReferenceForExportShardingTables

type GCSReferenceForExportShardingTables struct {
	// URI is Google Cloud Storage object
	URI string

	// DestinationFormat is the format to use when writing exported files.
	// Allowed values are: CSV, Avro, JSON.  The default is CSV.
	// CSV is not supported for tables with nested or repeated fields.
	DestinationFormat bigquery.DataFormat

	// Compression specifies the type of compression to apply when writing data
	// to Google Cloud Storage, or using this GCSReference as an ExternalData
	// source with CSV or JSON SourceFormat. Default is None.
	//
	// Avro files allow additional compression types: DEFLATE and SNAPPY.
	Compression bigquery.Compression
}

type GCSReferenceForExportShardingTablesReq

type GCSReferenceForExportShardingTablesReq struct {
	// URI is Google Cloud Storage object
	// {{TABLE_ID}} を入れるとこれはExportするTableIDに置き換えられます
	// 対象のTableが1GBを超えることを想定してWildcardを含めてください
	// eg. gs://hoge/{{TABLE_ID}}/*.parquest
	URI string `json:"uri"`

	// DestinationFormat is the format to use when writing exported files.
	// Allowed values are: CSV, Avro, JSON.  The default is CSV.
	// CSV is not supported for tables with nested or repeated fields.
	DestinationFormat string

	// Compression specifies the type of compression to apply when writing data
	// to Google Cloud Storage, or using this GCSReference as an ExternalData
	// source with CSV or JSON SourceFormat. Default is None.
	//
	// Avro files allow additional compression types: DEFLATE and SNAPPY.
	Compression string
}

type Service

type Service struct {
	BQ *bigquery.Client
}

func NewService

func NewService(ctx context.Context, bq *bigquery.Client) (*Service, error)

func (*Service) ExportShardingTable

func (s *Service) ExportShardingTable(ctx context.Context, to *GCSReferenceForExportShardingTables, projectID string, datasetID string, tableID string) (*bigquery.Job, error)

func (*Service) ExportShardingTables

func (s *Service) ExportShardingTables(ctx context.Context, to *GCSReferenceForExportShardingTables, projectID string, datasetID string, target *DateShardingTableTarget, jobFunc func(ctx context.Context, jobID string), limit *ExportShardingTablesLimit, ops ...APIOptions) ([]string, error)

type TargetTable

type TargetTable struct {
	Project       string `json:"project"`
	Dataset       string `json:"dataset"`
	TablePrefix   string `json:"tablePrefix"`
	ExpirationDay int    `json:"expirationDay"`
}