storage

Documentation

Index

• type GCSConfig
• type GCSConnector
  • func MustNewGCSConnector(ctx context.Context, cfg *GCSConfig, logger framework.Logger) *GCSConnector
  • func NewGCSConnector(ctx context.Context, cfg *GCSConfig) (*GCSConnector, error)
  • func (g *GCSConnector) Bucket() string
  • func (g *GCSConnector) ProjectID() string
  • func (g *GCSConnector) RangeSize() uint64
  • func (g *GCSConnector) WriteMany(ctx context.Context, input []interface{}, mapToStruct interface{}, ...) error
  • func (g *GCSConnector) WriteOne(ctx context.Context, input interface{}, mapToStruct interface{}, ...) error
• type Store

Types

type GCSConfig

type GCSConfig struct {
	BucketName string `env:"GCS_BUCKET_NAME,required"`
	ProjectID  string `env:"GCP_PROJECT_ID,required"`
	RangeSize  uint64 `env:"GCS_DIR_RANGE_SIZE" envDefault:"10000"`
}

type GCSConnector

type GCSConnector struct {
	// contains filtered or unexported fields
}

func MustNewGCSConnector

func MustNewGCSConnector(ctx context.Context, cfg *GCSConfig, logger framework.Logger) *GCSConnector

func NewGCSConnector

func NewGCSConnector(ctx context.Context, cfg *GCSConfig) (*GCSConnector, error)

func (*GCSConnector) Bucket

func (g *GCSConnector) Bucket() string

func (*GCSConnector) ProjectID

func (g *GCSConnector) ProjectID() string

func (*GCSConnector) RangeSize

func (g *GCSConnector) RangeSize() uint64

func (*GCSConnector) WriteMany

func (g *GCSConnector) WriteMany(ctx context.Context, input []interface{}, mapToStruct interface{}, filename string) error

Write writes a mutliple parquets to GCS storage

func (*GCSConnector) WriteOne

func (g *GCSConnector) WriteOne(ctx context.Context, input interface{}, mapToStruct interface{}, filename string) error

Write writes a single parquet to GCS storage

type Store

type Store interface {
	WriteOne(ctx context.Context, input interface{}, mapToStruct interface{}, filename string) error
	WriteMany(ctx context.Context, input []interface{}, mapToStruct interface{}, filename string) error
	ProjectID() string
	Bucket() string
	RangeSize() uint64
}