etl

Documentation

Index

• Constants
• func GetVectorArrayLen(ctx context.Context, vec *vector.Vector) (int, error)
• func ValToString(ctx context.Context, vec *vector.Vector, rowIdx int) (string, error)
• type BufWriter
  • func NewBufWriter(ctx context.Context, writer io.Writer) *BufWriter
  • func (w *BufWriter) Close() error
  • func (w *BufWriter) Write(p []byte) (n int, err error)
• type CSVWriter
  • func NewCSVWriter(ctx context.Context, writer io.StringWriter) *CSVWriter
  • func (w *CSVWriter) FlushAndClose() (int, error)
  • func (w *CSVWriter) GetContent() string
  • func (w *CSVWriter) WriteRow(row *table.Row) error
  • func (w *CSVWriter) WriteStrings(record []string) error
• type DefaultSqlWriter
  • func NewSqlWriter(ctx context.Context, tbl *table.Table, csv *CSVWriter) *DefaultSqlWriter
  • func (sw *DefaultSqlWriter) FlushAndClose() (int, error)
  • func (sw *DefaultSqlWriter) GetContent() string
  • func (sw *DefaultSqlWriter) WriteRow(row *table.Row) error
  • func (sw *DefaultSqlWriter) WriteStrings(record []string) error
• type FSWriter
  • func NewFSWriter(ctx context.Context, fs fileservice.FileService, opts ...FSWriterOption) *FSWriter
  • func (w *FSWriter) Write(p []byte) (n int, err error)
  • func (w *FSWriter) WriteString(s string) (n int, err error)
• type FSWriterOption
  • func WithFilePath(filepath string) FSWriterOption
  • func (f FSWriterOption) Apply(w *FSWriter)
• type TAEReader
  • func NewTaeReader(ctx context.Context, tbl *table.Table, filePath string, filesize int64, ...) (*TAEReader, error)
  • func (r *TAEReader) Close()
  • func (r *TAEReader) ReadAll(ctx context.Context) ([]*batch.Batch, error)
  • func (r *TAEReader) ReadLine() ([]string, error)
  • func (r *TAEReader) ReadRow(row *table.Row) error
• type TAEWriter
  • func NewTAEWriter(ctx context.Context, tbl *table.Table, mp *mpool.MPool, filePath string, ...) *TAEWriter
  • func (w *TAEWriter) FlushAndClose() (int, error)
  • func (w *TAEWriter) GetContent() string
  • func (w *TAEWriter) WriteRow(row *table.Row) error
  • func (w *TAEWriter) WriteStrings(Line []string) error

Constants

const BatchSize = 8192

const MAX_INSERT_TIME = 3 * time.Second

Functions

func GetVectorArrayLen

func GetVectorArrayLen(ctx context.Context, vec *vector.Vector) (int, error)

func ValToString

func ValToString(ctx context.Context, vec *vector.Vector, rowIdx int) (string, error)

Types

type BufWriter

type BufWriter struct {
	// contains filtered or unexported fields
}

func NewBufWriter

func NewBufWriter(ctx context.Context, writer io.Writer) *BufWriter

func (*BufWriter) Close

func (w *BufWriter) Close() error

func (*BufWriter) Write

func (w *BufWriter) Write(p []byte) (n int, err error)

type CSVWriter

type CSVWriter struct {
	// contains filtered or unexported fields
}

func NewCSVWriter

func NewCSVWriter(ctx context.Context, writer io.StringWriter) *CSVWriter

func (*CSVWriter) FlushAndClose

func (w *CSVWriter) FlushAndClose() (int, error)

func (*CSVWriter) GetContent

func (w *CSVWriter) GetContent() string

func (*CSVWriter) WriteRow

func (w *CSVWriter) WriteRow(row *table.Row) error

func (*CSVWriter) WriteStrings

func (w *CSVWriter) WriteStrings(record []string) error

type DefaultSqlWriter

type DefaultSqlWriter struct {
	// contains filtered or unexported fields
}

DefaultSqlWriter SqlWriter is a writer that writes data to a SQL database.

func NewSqlWriter

func NewSqlWriter(ctx context.Context, tbl *table.Table, csv *CSVWriter) *DefaultSqlWriter

func (*DefaultSqlWriter) FlushAndClose

func (sw *DefaultSqlWriter) FlushAndClose() (int, error)

func (*DefaultSqlWriter) GetContent

func (sw *DefaultSqlWriter) GetContent() string

func (*DefaultSqlWriter) WriteRow

func (sw *DefaultSqlWriter) WriteRow(row *table.Row) error

func (*DefaultSqlWriter) WriteStrings

func (sw *DefaultSqlWriter) WriteStrings(record []string) error

type FSWriter

type FSWriter struct {
	// contains filtered or unexported fields
}

func NewFSWriter

func NewFSWriter(ctx context.Context, fs fileservice.FileService, opts ...FSWriterOption) *FSWriter

func (*FSWriter) Write

func (w *FSWriter) Write(p []byte) (n int, err error)

Write implement io.Writer, Please execute in series

func (*FSWriter) WriteString

func (w *FSWriter) WriteString(s string) (n int, err error)

WriteString implement io.StringWriter

type FSWriterOption

type FSWriterOption func(*FSWriter)

func WithFilePath

func WithFilePath(filepath string) FSWriterOption

func (FSWriterOption) Apply

func (f FSWriterOption) Apply(w *FSWriter)

type TAEReader

type TAEReader struct {
	// contains filtered or unexported fields
}

TAEReader implements the io.Reader interface for reading a tae file. Deprecated

func NewTaeReader

func NewTaeReader(ctx context.Context, tbl *table.Table, filePath string, filesize int64, fs fileservice.FileService, mp *mpool.MPool) (*TAEReader, error)

NewTaeReader returns a TAEReader. Deprecated

func (*TAEReader) Close

func (r *TAEReader) Close()

func (*TAEReader) ReadAll

func (r *TAEReader) ReadAll(ctx context.Context) ([]*batch.Batch, error)

func (*TAEReader) ReadLine

func (r *TAEReader) ReadLine() ([]string, error)

func (*TAEReader) ReadRow

func (r *TAEReader) ReadRow(row *table.Row) error

type TAEWriter

type TAEWriter struct {
	// contains filtered or unexported fields
}

TAEWriter implements table.RowWriter and writes data to a tae file. Deprecated

func NewTAEWriter

func NewTAEWriter(ctx context.Context, tbl *table.Table, mp *mpool.MPool, filePath string, fs fileservice.FileService) *TAEWriter

NewTAEWriter returns a new instance of TAEWriter Deprecated

func (*TAEWriter) FlushAndClose

func (w *TAEWriter) FlushAndClose() (int, error)

FlushAndClose implement ETLWriter

func (*TAEWriter) GetContent

func (w *TAEWriter) GetContent() string

func (*TAEWriter) WriteRow

func (w *TAEWriter) WriteRow(row *table.Row) error

WriteRow implement ETLWriter

func (*TAEWriter) WriteStrings

func (w *TAEWriter) WriteStrings(Line []string) error

WriteStrings implement ETLWriter