Documentation ¶
Index ¶
- Constants
- Variables
- type AWS
- type App
- type Base64String
- type Cloud
- type Config
- type Factory
- func (f *Factory) NewAWSSession() *session.Session
- func (f *Factory) NewApp() *App
- func (f *Factory) NewBigQueryOptions() []option.ClientOption
- func (f *Factory) NewCloudStorageOptions() []option.ClientOption
- func (f *Factory) NewGCPOptions() []option.ClientOption
- func (f *Factory) NewLoader() *Loader
- func (f *Factory) NewReceiver() *Receiver
- func (f *Factory) NewResolver() *Resolver
- func (f *Factory) NewTransporter() *Transporter
- type GCP
- type Job
- type JobOption
- type Loader
- type LoadingDestination
- type LoadingJob
- type ReceiptHandle
- type Receiver
- type Resolver
- type Rule
- type RunOption
- type RunSettings
- type S3Object
- type S3Soruce
- type SourceFormat
- type TransportJob
- type TransportJobHandle
- type Transporter
Constants ¶
View Source
const ( S3URITemplate = "s3://%s/%s" BigQueryTableTemplate = "%s.%s.%s" )
View Source
const ( Unknown SourceFormat = "" CSV = "csv" JSON = "json" Parquet = "parquet" )
Variables ¶
View Source
var ( ErrMaxRetry = errors.New("max retry count reached") ErrNoMessage = errors.New("no sqs message") )
View Source
var ErrInvalidHandle = errors.New("invalid handle")
Functions ¶
This section is empty.
Types ¶
type AWS ¶ added in v0.3.0
type AWS struct { Region string `yaml:"region,omitempty"` DisableSSL bool `yaml:"disable_ssl,omitempty"` S3ForcePathStyle bool `yaml:"s3_force_path_style,omitempty"` S3Endpoint string `yaml:"s3_endpoint,omitempty"` SQSEndpoint string `yaml:"sqs_endpoint,omitempty"` AccessKeyID string `yaml:"access_key_id,omitempty"` SecretAccessKey string `yaml:"secret_access_key,omitempty"` DisableShardConfigState bool `yaml:"disable_shard_config_state,omitempty"` }
type Base64String ¶ added in v0.3.0
type Base64String []byte
func (Base64String) Bytes ¶ added in v0.3.0
func (s Base64String) Bytes() []byte
func (Base64String) IsEmpty ¶ added in v0.3.0
func (s Base64String) IsEmpty() bool
func (Base64String) String ¶ added in v0.3.0
func (s Base64String) String() string
func (*Base64String) UnmarshalYAML ¶ added in v0.3.0
func (s *Base64String) UnmarshalYAML(unmarshal func(interface{}) error) (err error)
type Config ¶
type Config struct { QueueName string `yaml:"queue_name"` Cloud *Cloud `yaml:"cloud"` Rules []*Rule `yaml:"rules"` Rule `yaml:",inline"` }
func LoadConfig ¶
func NewDefaultConfig ¶ added in v0.3.0
func NewDefaultConfig() *Config
type Factory ¶ added in v0.3.0
type Factory struct {
*Config
}
func (*Factory) NewAWSSession ¶ added in v0.3.0
func (*Factory) NewBigQueryOptions ¶ added in v0.3.0
func (f *Factory) NewBigQueryOptions() []option.ClientOption
func (*Factory) NewCloudStorageOptions ¶ added in v0.3.0
func (f *Factory) NewCloudStorageOptions() []option.ClientOption
func (*Factory) NewGCPOptions ¶ added in v0.3.0
func (f *Factory) NewGCPOptions() []option.ClientOption
func (*Factory) NewReceiver ¶ added in v0.3.0
func (*Factory) NewResolver ¶ added in v0.3.0
func (*Factory) NewTransporter ¶ added in v0.3.0
func (f *Factory) NewTransporter() *Transporter
type GCP ¶ added in v0.3.0
type GCP struct { WithoutAuthentication bool `yaml:"without_authentication,omitempty"` BigQueryEndpoint string `yaml:"big_query_endpoint,omitempty"` CloudStorageEndpoint string `yaml:"cloud_storage_endpoint,omitempty"` Base64Credential Base64String `yaml:"base64_credential"` }
type Job ¶ added in v0.3.0
type Job struct { *TransportJob *LoadingJob }
type JobOption ¶ added in v0.3.0
type JobOption struct { TemporaryBucket string `yaml:"temporary_bucket" json:"temporary_bucket"` GZip *bool `yaml:"gzip,omitempty" json:"gzip,omitempty"` AutoDetect *bool `yaml:"auto_detect,omitempty" json:"auto_detect,omitempty"` SourceFormat SourceFormat `yaml:"source_format" json:"source_format"` }
type Loader ¶ added in v0.3.0
type Loader struct {
// contains filtered or unexported fields
}
func NewLoader ¶ added in v0.3.0
func NewLoader(opts ...option.ClientOption) *Loader
type LoadingDestination ¶ added in v0.3.0
type LoadingDestination struct { ProjectID string `yaml:"project_id" json:"project_id"` Dataset string `yaml:"dataset" json:"dataset"` Table string `yaml:"table" json:"table"` }
func (*LoadingDestination) Clone ¶ added in v0.3.0
func (bq *LoadingDestination) Clone() *LoadingDestination
func (*LoadingDestination) MergeIn ¶ added in v0.3.0
func (bq *LoadingDestination) MergeIn(other *LoadingDestination)
func (LoadingDestination) String ¶ added in v0.3.0
func (bq LoadingDestination) String() string
type LoadingJob ¶ added in v0.3.0
type LoadingJob struct { GCSRef *bigquery.GCSReference *LoadingDestination CreateDisposition bigquery.TableCreateDisposition WriteDisposition bigquery.TableWriteDisposition }
func NewLoadingJob ¶ added in v0.3.0
func NewLoadingJob(dest *LoadingDestination, objectURIs ...string) *LoadingJob
func (*LoadingJob) String ¶ added in v0.3.0
func (job *LoadingJob) String() string
type ReceiptHandle ¶ added in v0.3.0
type ReceiptHandle struct {
// contains filtered or unexported fields
}
func (*ReceiptHandle) Cleanup ¶ added in v0.3.0
func (h *ReceiptHandle) Cleanup()
func (*ReceiptHandle) Complete ¶ added in v0.3.0
func (h *ReceiptHandle) Complete() error
func (*ReceiptHandle) Debugf ¶ added in v0.3.0
func (h *ReceiptHandle) Debugf(format string, args ...interface{})
func (*ReceiptHandle) Errorf ¶ added in v0.3.0
func (h *ReceiptHandle) Errorf(format string, args ...interface{})
func (*ReceiptHandle) Infof ¶ added in v0.3.0
func (h *ReceiptHandle) Infof(format string, args ...interface{})
type Receiver ¶
type Receiver struct {
// contains filtered or unexported fields
}
func NewReceiver ¶ added in v0.3.0
func (*Receiver) GetQueueName ¶ added in v0.3.0
func (*Receiver) SetQueueName ¶ added in v0.3.0
type Resolver ¶ added in v0.3.0
type Resolver struct {
// contains filtered or unexported fields
}
func NewResolver ¶ added in v0.3.0
type Rule ¶
type Rule struct { S3 *S3Soruce `yaml:"s3"` BigQuery *LoadingDestination `yaml:"big_query"` Option *JobOption `yaml:"option"` // contains filtered or unexported fields }
type RunOption ¶ added in v0.3.0
type RunOption interface {
Apply(*RunSettings)
}
func WithExitError ¶ added in v0.3.0
func WithExitNoMessage ¶ added in v0.3.0
func WithQueueName ¶ added in v0.3.0
type RunSettings ¶ added in v0.3.0
func (*RunSettings) Apply ¶ added in v0.3.0
func (s *RunSettings) Apply(o *RunSettings)
type S3Soruce ¶
type SourceFormat ¶ added in v0.2.0
type SourceFormat string
func (SourceFormat) Is ¶ added in v0.2.0
func (f SourceFormat) Is(others ...SourceFormat) bool
func (SourceFormat) IsSupport ¶ added in v0.2.0
func (f SourceFormat) IsSupport() bool
type TransportJob ¶ added in v0.3.0
func (*TransportJob) String ¶ added in v0.3.0
func (job *TransportJob) String() string
type TransportJobHandle ¶ added in v0.3.0
type TransportJobHandle struct {
// contains filtered or unexported fields
}
type Transporter ¶ added in v0.3.0
type Transporter struct {
// contains filtered or unexported fields
}
func NewTransporter ¶ added in v0.3.0
func NewTransporter(sess *session.Session, opts ...option.ClientOption) *Transporter
func (*Transporter) Transport ¶ added in v0.3.0
func (t *Transporter) Transport(ctx context.Context, job *TransportJob) (*TransportJobHandle, error)
Source Files ¶
Click to show internal directories.
Click to hide internal directories.