bqworker

Jul 27, 2022

Changes in this version

type OptionFunc

+ func WithMaxRetry(retry int) OptionFunc

Jul 8, 2022

Changes in this version

+ const ChecksumFieldName

Jul 8, 2022

Changes in this version

type TableSchema

+ func (ts *TableSchema) ChecksumField() bool

Jul 7, 2022

Changes in this version

type TableSchema

+ ValueFromExtra func(k string) (interface{}, bool)

type TableSchemaGroup

+ TableSchemaList func() []TableSchema

Jun 30, 2022

Changes in this version

type TableSchemaGroup

+ func NewTableSchemaGroup(projectId, credential string, schemas []*TableSchema) (TableSchemaGroup, error)

+ Credential func() string

Jun 29, 2022

Changes in this version

+ var ErrNotFoundBigQueryClient = errors.New("[err] not found bigquery client")

+ var ErrNotFoundBigQueryTableSchema = errors.New("[err] not found bigquery table schema")

type Message

+ ProjectId string

+ type Option interface

+ type OptionFunc func(cfg *config)

+ func WithDispatcherLoopWaitDuration(duration time.Duration) OptionFunc

+ func WithErrorHandler(f ErrorHandler) OptionFunc

+ func WithQueueSize(size int) OptionFunc

+ func WithWorkerSize(size int) OptionFunc

+ func WithWorkerStack(stack int) OptionFunc

+ func WithWorkerWaitDuration(duration time.Duration) OptionFunc

type Row

+ ProjectId func() string

type Streamer

+ Start func() error

+ Stop func() error

type TableSchema

+ ProjectId string

+ func NewTableSchema(projectId string, datasetId, prefix string, meta *bigquery.TableMetadata, ...) (*TableSchema, error)

+ func (ts *TableSchema) Valid() bool

+ type TableSchemaGroup interface

+ Jwt func() string

+ ProjectId func() string

+ TableSchemas func() []*TableSchema

Nov 11, 2021

Changes in this version

type TableSchema

+ AddNewColumns bool

Feb 19, 2020

Changes in this version

+ var ErrInvalidParams = errors.New("[err] invalid params")

+ var ErrTimeout = errors.New("[err] timeout")

+ type Config struct

+ func NewConfig(projectId string, jwtbys []byte, schemas []*TableSchema, ...) (*Config, error)

+ type ErrorHandler func(error)

+ type Job struct

+ Msg *Message

+ type Message struct

+ Data Row

+ DatasetId string

+ TableId string

+ type Row interface

+ InsertId func() string

+ PublishedAt func() time.Time

+ Save func() (row map[string]bigquery.Value, insertID string, err error)

+ Schema func() (schema *TableSchema, err error)

+ type Streamer interface

+ AddRow func(ctx context.Context, row Row) error

+ AddRowSync func(ctx context.Context, row Row) error

+ func NewStreamer(cfg *Config, errFunc ErrorHandler) (Streamer, error)

+ type TablePeriod int

+ const Daily

+ const Monthly

+ const NotExist

+ const Yearly

+ type TableSchema struct

+ DatasetId string

+ Meta *bigquery.TableMetadata

+ Period TablePeriod

+ Prefix string

+ type Worker struct

+ type WorkerDispatcher struct