file

package
v0.19.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 11, 2025 License: Apache-2.0 Imports: 32 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func AddFileWriter added in v0.11.0 

func AddFileWriter(format string, ds *data.DataStream, writer io.WriteCloser) (data.DataWriter, error)

func BuildFullPath added in v0.10.1 

func BuildFullPath(parsed *url.URL, filename string) (*url.URL, error)

func CreateCompressedWriter added in v0.11.0 

func CreateCompressedWriter(bufWriter io.WriteCloser, compressionType string, format string) (io.WriteCloser, error)

func GetIo 

func GetIo(ctx context.Context, counting io.WriteCloser, filePath *url.URL) (io.WriteCloser, error)

func GetPathAndIO added in v0.11.0 

func GetPathAndIO(ctx context.Context, path *url.URL, counter io.WriteCloser, compression, format string) (io.WriteCloser, error)

func NewProgressBar added in v0.11.1 

func NewProgressBar() *progressbar.ProgressBar

func ValueToString added in v0.15.0 

func ValueToString(value any, col data.Column) (string, error)

Types

type AzureBlob added in v0.16.0 

type AzureBlob struct {
	*AzureBlobConfig
	// contains filtered or unexported fields
}

func (*AzureBlob) Close added in v0.16.0 

func (a *AzureBlob) Close() error

func (*AzureBlob) Write added in v0.16.0 

func (a *AzureBlob) Write(p []byte) (n int, err error)

type AzureBlobConfig added in v0.16.0 

type AzureBlobConfig struct {
	// contains filtered or unexported fields
}

func ParseAzureBlobURL added in v0.16.0 

func ParseAzureBlobURL(blobUrl *url.URL) (*AzureBlobConfig, error)

func ParseAzurite added in v0.16.0 

func ParseAzurite(blobUrl *url.URL) (*AzureBlobConfig, error)

func (*AzureBlobConfig) GetWriter added in v0.16.0 

func (a *AzureBlobConfig) GetWriter(ctx context.Context) (*AzureBlob, error)

type BufferedWriter added in v0.16.0 

type BufferedWriter struct {
	// contains filtered or unexported fields
}

func NewBufferedWriter added in v0.16.0 

func NewBufferedWriter(writer io.Writer, closers ...io.WriteCloser) *BufferedWriter

func (*BufferedWriter) Close added in v0.16.0 

func (b *BufferedWriter) Close() error

func (*BufferedWriter) Write added in v0.16.0 

func (b *BufferedWriter) Write(p []byte) (n int, err error)

type CSVBatchWriter added in v0.13.0 

type CSVBatchWriter struct {
	// contains filtered or unexported fields
}

func (*CSVBatchWriter) WriteBatch added in v0.13.0 

func (cb *CSVBatchWriter) WriteBatch(batch data.Batch) error

type CSVDataWriter 

type CSVDataWriter struct {
	// contains filtered or unexported fields
}

func AddCSV 

func AddCSV(ds *data.DataStream, writer io.WriteCloser) *CSVDataWriter

func NewCSVDataWriter 

func NewCSVDataWriter(datastream *data.DataStream, writer io.WriteCloser) *CSVDataWriter

func (*CSVDataWriter) Close 

func (cw *CSVDataWriter) Close() error

func (*CSVDataWriter) CreateBatchWriter added in v0.13.0 

func (cw *CSVDataWriter) CreateBatchWriter() data.BatchWriter

func (*CSVDataWriter) Flush 

func (cw *CSVDataWriter) Flush() error

func (*CSVDataWriter) ProcessRow added in v0.13.0 

func (cw *CSVDataWriter) ProcessRow(row []any) ([]string, error)

type ColumnMetadata added in v0.18.0 

type ColumnMetadata struct {
	Name string `json:"name"`
	Type string `json:"type"`
	Len  int32  `json:"len"`
}

type DecimalType added in v0.6.0 

type DecimalType int
const (
	UnknownDecimal DecimalType = iota
	Int32Decimal
	Int64Decimal
	ByteArrayDecimal
)

type JSONLBatchWriter added in v0.13.0 

type JSONLBatchWriter struct {
	// contains filtered or unexported fields
}

func (*JSONLBatchWriter) WriteBatch added in v0.13.0 

func (bw *JSONLBatchWriter) WriteBatch(batch data.Batch) error

type JSONLWriter 

type JSONLWriter struct {
	// contains filtered or unexported fields
}

func AddJSONL 

func AddJSONL(ds *data.DataStream, writer io.WriteCloser) *JSONLWriter

func NewJSONLWriter added in v0.13.0 

func NewJSONLWriter(ds *data.DataStream, writer io.Writer) *JSONLWriter

func (*JSONLWriter) Close 

func (w *JSONLWriter) Close() error

func (*JSONLWriter) CreateBatchWriter added in v0.13.0 

func (w *JSONLWriter) CreateBatchWriter() data.BatchWriter

func (*JSONLWriter) Flush 

func (w *JSONLWriter) Flush() error

func (*JSONLWriter) ProcessRow added in v0.13.0 

func (w *JSONLWriter) ProcessRow(row []any) ([]byte, error)

type MappedType added in v0.6.0 

type MappedType struct {
	ParquetType parquet.Type
	LogicalType schema.LogicalType
	GoType      reflect.Type
}

type ParquetBatchWriter added in v0.13.0 

type ParquetBatchWriter struct {
	// contains filtered or unexported fields
}

func (*ParquetBatchWriter) WriteBatch added in v0.13.0 

func (pb *ParquetBatchWriter) WriteBatch(batch data.Batch) error

type ParquetDataWriter 

type ParquetDataWriter struct {
	// contains filtered or unexported fields
}

func AddParquet 

func AddParquet(ds *data.DataStream, writer io.WriteCloser) *ParquetDataWriter

func NewParquetDataWriter 

func NewParquetDataWriter(datastream *data.DataStream, ioWriter io.Writer) *ParquetDataWriter

func (*ParquetDataWriter) Close 

func (pw *ParquetDataWriter) Close() error

func (*ParquetDataWriter) CreateBatchWriter added in v0.13.0 

func (pw *ParquetDataWriter) CreateBatchWriter() data.BatchWriter

func (*ParquetDataWriter) Flush 

func (pw *ParquetDataWriter) Flush() error

type ParquetDecimal added in v0.6.0 

type ParquetDecimal struct {
	Type         DecimalType // Discriminator to indicate the active type
	Int32Val     int32
	Int64Val     int64
	ByteArrayVal []byte
}

type WriteCloseBuffer added in v0.16.0 

type WriteCloseBuffer struct {
	*bytes.Buffer
}

func NewWriteCloseBuffer added in v0.16.0 

func NewWriteCloseBuffer(buf *bytes.Buffer) *WriteCloseBuffer

NewWriteCloseBuffer initializes and returns a new WriteCloseBuffer.

func (*WriteCloseBuffer) Close added in v0.16.0 

func (wcb *WriteCloseBuffer) Close() error

func (*WriteCloseBuffer) Write added in v0.16.0 

func (wcb *WriteCloseBuffer) Write(p []byte) (n int, err error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.