file

Documentation

Index

• func AddFileWriter(format string, ds *data.DataStream, writer io.Writer) (data.DataWriter, error)
• func BuildFullPath(parsed *url.URL, filename string) (*url.URL, error)
• func CreateCompressedWriter(bufWriter *bufio.Writer, compressionType string, format string) (io.WriteCloser, error)
• func GetIo(counting io.Writer, filePath *url.URL) (*bufio.Writer, error)
• func GetPathAndIO(parsed *url.URL, counter io.Writer, filename, compression, format string) (*url.URL, io.WriteCloser, error)
• func NewProgressBar() *progressbar.ProgressBar
• type CSVBatchWriter
  • func (cb *CSVBatchWriter) WriteBatch(batch data.Batch) error
• type CSVDataWriter
  • func AddCSV(ds *data.DataStream, writer io.Writer) *CSVDataWriter
  • func NewCSVDataWriter(datastream *data.DataStream, writer io.Writer) *CSVDataWriter
  • func (cw *CSVDataWriter) Close() error
  • func (cw *CSVDataWriter) CreateBatchWriter() data.BatchWriter
  • func (cw *CSVDataWriter) Flush() error
  • func (cw *CSVDataWriter) ProcessRow(row []any) ([]string, error)
• type DecimalType
• type JSONLBatchWriter
  • func (bw *JSONLBatchWriter) WriteBatch(batch data.Batch) error
• type JSONLWriter
  • func AddJSONL(ds *data.DataStream, writer io.Writer) *JSONLWriter
  • func NewJSONLWriter(ds *data.DataStream, writer io.Writer) *JSONLWriter
  • func (w *JSONLWriter) Close() error
  • func (w *JSONLWriter) CreateBatchWriter() data.BatchWriter
  • func (w *JSONLWriter) Flush() error
  • func (w *JSONLWriter) ProcessRow(row []any) ([]byte, error)
• type MappedType
• type ParquetBatchWriter
  • func (pb *ParquetBatchWriter) WriteBatch(batch data.Batch) error
• type ParquetDataWriter
  • func AddParquet(ds *data.DataStream, writer io.Writer) *ParquetDataWriter
  • func NewParquetDataWriter(datastream *data.DataStream, ioWriter io.Writer) *ParquetDataWriter
  • func (pw *ParquetDataWriter) Close() error
  • func (pw *ParquetDataWriter) CreateBatchWriter() data.BatchWriter
  • func (pw *ParquetDataWriter) Flush() error
• type ParquetDecimal

Functions

func AddFileWriter

func AddFileWriter(format string, ds *data.DataStream, writer io.Writer) (data.DataWriter, error)

func BuildFullPath

func BuildFullPath(parsed *url.URL, filename string) (*url.URL, error)

func CreateCompressedWriter

func CreateCompressedWriter(bufWriter *bufio.Writer, compressionType string, format string) (io.WriteCloser, error)

func GetIo

func GetIo(counting io.Writer, filePath *url.URL) (*bufio.Writer, error)

func GetPathAndIO

func GetPathAndIO(parsed *url.URL, counter io.Writer, filename, compression, format string) (*url.URL, io.WriteCloser, error)

func NewProgressBar

func NewProgressBar() *progressbar.ProgressBar

Types

type CSVBatchWriter

type CSVBatchWriter struct {
	// contains filtered or unexported fields
}

func (*CSVBatchWriter) WriteBatch

func (cb *CSVBatchWriter) WriteBatch(batch data.Batch) error

type CSVDataWriter

type CSVDataWriter struct {
	// contains filtered or unexported fields
}

func AddCSV

func AddCSV(ds *data.DataStream, writer io.Writer) *CSVDataWriter

func NewCSVDataWriter

func NewCSVDataWriter(datastream *data.DataStream, writer io.Writer) *CSVDataWriter

func (*CSVDataWriter) Close

func (cw *CSVDataWriter) Close() error

func (*CSVDataWriter) CreateBatchWriter

func (cw *CSVDataWriter) CreateBatchWriter() data.BatchWriter

func (*CSVDataWriter) Flush

func (cw *CSVDataWriter) Flush() error

func (*CSVDataWriter) ProcessRow

func (cw *CSVDataWriter) ProcessRow(row []any) ([]string, error)

type DecimalType

type DecimalType int

const (
	UnknownDecimal DecimalType = iota
	Int32Decimal
	Int64Decimal
	ByteArrayDecimal
)

type JSONLBatchWriter

type JSONLBatchWriter struct {
	// contains filtered or unexported fields
}

func (*JSONLBatchWriter) WriteBatch

func (bw *JSONLBatchWriter) WriteBatch(batch data.Batch) error

type JSONLWriter

type JSONLWriter struct {
	// contains filtered or unexported fields
}

func AddJSONL

func AddJSONL(ds *data.DataStream, writer io.Writer) *JSONLWriter

func NewJSONLWriter

func NewJSONLWriter(ds *data.DataStream, writer io.Writer) *JSONLWriter

func (*JSONLWriter) Close

func (w *JSONLWriter) Close() error

func (*JSONLWriter) CreateBatchWriter

func (w *JSONLWriter) CreateBatchWriter() data.BatchWriter

func (*JSONLWriter) Flush

func (w *JSONLWriter) Flush() error

func (*JSONLWriter) ProcessRow

func (w *JSONLWriter) ProcessRow(row []any) ([]byte, error)

type MappedType

type MappedType struct {
	ParquetType parquet.Type
	LogicalType schema.LogicalType
	GoType      reflect.Type
}

type ParquetBatchWriter

type ParquetBatchWriter struct {
	// contains filtered or unexported fields
}

func (*ParquetBatchWriter) WriteBatch

func (pb *ParquetBatchWriter) WriteBatch(batch data.Batch) error

type ParquetDataWriter

type ParquetDataWriter struct {
	// contains filtered or unexported fields
}

func AddParquet

func AddParquet(ds *data.DataStream, writer io.Writer) *ParquetDataWriter

func NewParquetDataWriter

func NewParquetDataWriter(datastream *data.DataStream, ioWriter io.Writer) *ParquetDataWriter

func (*ParquetDataWriter) Close

func (pw *ParquetDataWriter) Close() error

func (*ParquetDataWriter) CreateBatchWriter

func (pw *ParquetDataWriter) CreateBatchWriter() data.BatchWriter

func (*ParquetDataWriter) Flush

func (pw *ParquetDataWriter) Flush() error

type ParquetDecimal

type ParquetDecimal struct {
	Type         DecimalType // Discriminator to indicate the active type
	Int32Val     int32
	Int64Val     int64
	ByteArrayVal []byte
}