formats

Documentation

Index

• Variables
• type S3Parquet
  • func NewS3Parquet(config S3ParquetConfig) (*S3Parquet, error)
  • func (f *S3Parquet) NewReader(ctx context.Context, path string) (core.SegmentReader, error)
  • func (f *S3Parquet) NewWriter(ctx context.Context, path string) (core.SegmentWriter, error)
• type S3ParquetConfig
  • func (c S3ParquetConfig) Get() (interface{}, error)

Variables

var (
	// DefaultParquetParallelism is the default Parquet library parallelism parameter for read/write operations.
	DefaultParquetParallelism = 10

	// DefaultParquetCreatedBy is the default value used for Parquet file CreatedBy attribute.
	DefaultParquetCreatedBy = "kafka-replicator"
)

Types

type S3Parquet

type S3Parquet struct {
	// contains filtered or unexported fields
}

S3Parquet is the segment format backed by AWS S3 and Parquet columnar data storage format

func NewS3Parquet

func NewS3Parquet(config S3ParquetConfig) (*S3Parquet, error)

NewS3Parquet returns a new S3Parquet instance

func (*S3Parquet) NewReader

func (f *S3Parquet) NewReader(ctx context.Context, path string) (core.SegmentReader, error)

NewReader creates a new segment reader Expected path format: s3://mybucket/mykey

func (*S3Parquet) NewWriter

func (f *S3Parquet) NewWriter(ctx context.Context, path string) (core.SegmentWriter, error)

NewWriter creates a new segment writer Expected path format: s3://mybucket/mykey

type S3ParquetConfig

type S3ParquetConfig struct {
	// The AWS config object used to create the AWS S3 client.
	//
	// Field value is required.
	AWSConfig *aws.Config `required:"true"`

	// Allows configuration of AWS S3 uploader instance.
	//
	// Field value is optional.
	S3UploaderOptions []func(*s3manager.Uploader)

	// Parquet library parallelism parameter for read/write operations.
	//
	// Default value is set via DefaultParquetParallelism variable.
	Parallelism int `min:"1"`

	// Is the value used for Parquet file CreatedBy attribute.
	//
	// Default value is set via DefaultParquetCreatedBy variable.
	CreatedBy string

	// Breaker enables tracking consumer error rate.
	//
	// Default value is set via DefaultS3Breaker variable.
	Breaker core.Breaker
}

S3ParquetConfig is the configuration for S3Parquet format

func (S3ParquetConfig) Get

func (c S3ParquetConfig) Get() (interface{}, error)

Get creates and returns the corresponding instance