external

Documentation

Index

• Constants
• Variables
• func Call(idx int, proc *process.Process, arg any, isFirst bool, isLast bool) (bool, error)
• func ColumnCntLargerErrorInfo() string
• func FilterFileList(node *plan.Node, proc *process.Process, fileList []string, fileSize []int64) ([]string, []int64, error)
• func GetBatchData(param *ExternalParam, plh *ParseLineHandler, proc *process.Process) (*batch.Batch, error)
• func IsSysTable(dbName string, tableName string) bool
• func Prepare(proc *process.Process, arg any) error
• func ReadFile(param *ExternalParam, proc *process.Process) (io.ReadCloser, error)
• func ReadFileOffset(param *tree.ExternParam, proc *process.Process, mcpu int, fileSize int64) ([][2]int, error)
• func ScanCsvFile(param *ExternalParam, proc *process.Process) (*batch.Batch, error)
• func ScanFileData(param *ExternalParam, proc *process.Process) (*batch.Batch, error)
• func ScanZonemapFile(param *ExternalParam, proc *process.Process) (*batch.Batch, error)
• func String(arg any, buf *bytes.Buffer)
• type Argument
  • func (arg *Argument) Free(proc *process.Process, pipelineFailed bool)
• type ExFileparam
• type ExParam
• type ExParamConst
• type ExternalParam
• type FilterParam
• type ParseLineHandler
  • func GetSimdcsvReader(param *ExternalParam, proc *process.Process) (*ParseLineHandler, error)
• type ZonemapFileparam

Constants

const NULL_FLAG = "\\N"

Variables

var (
	ONE_BATCH_MAX_ROW  = 40000
	S3_PARALLEL_MAXNUM = 10
)

var (
	STATEMENT_ACCOUNT = "account"
)

Functions

func Call

func Call(idx int, proc *process.Process, arg any, isFirst bool, isLast bool) (bool, error)

func ColumnCntLargerErrorInfo

func ColumnCntLargerErrorInfo() string

func FilterFileList

func FilterFileList(node *plan.Node, proc *process.Process, fileList []string, fileSize []int64) ([]string, []int64, error)

func GetBatchData

func GetBatchData(param *ExternalParam, plh *ParseLineHandler, proc *process.Process) (*batch.Batch, error)

func IsSysTable

func IsSysTable(dbName string, tableName string) bool

func Prepare

func Prepare(proc *process.Process, arg any) error

func ReadFile

func ReadFile(param *ExternalParam, proc *process.Process) (io.ReadCloser, error)

func ReadFileOffset

func ReadFileOffset(param *tree.ExternParam, proc *process.Process, mcpu int, fileSize int64) ([][2]int, error)

func ScanCsvFile

func ScanCsvFile(param *ExternalParam, proc *process.Process) (*batch.Batch, error)

func ScanFileData

func ScanFileData(param *ExternalParam, proc *process.Process) (*batch.Batch, error)

ScanFileData read batch data from external file

func ScanZonemapFile

func ScanZonemapFile(param *ExternalParam, proc *process.Process) (*batch.Batch, error)

func String

func String(arg any, buf *bytes.Buffer)

Types

type Argument

type Argument struct {
	Es *ExternalParam
}

func (*Argument) Free

func (arg *Argument) Free(proc *process.Process, pipelineFailed bool)

type ExFileparam

type ExFileparam struct {
	End       bool
	FileCnt   int
	FileFin   int
	FileIndex int
	Filepath  string
}

type ExParam

type ExParam struct {
	Fileparam *ExFileparam
	Zoneparam *ZonemapFileparam
	Filter    *FilterParam
	// contains filtered or unexported fields
}

type ExParamConst

type ExParamConst struct {
	IgnoreLine    int
	IgnoreLineTag int

	CreateSql     string
	Attrs         []string
	Cols          []*plan.ColDef
	OriginCols    []*plan.ColDef
	FileList      []string
	FileSize      []int64
	FileOffset    [][2]int
	Name2ColIndex map[string]int32
	Ctx           context.Context
	Extern        *tree.ExternParam

	ClusterTable *plan.ClusterTable
	// contains filtered or unexported fields
}

type ExternalParam

type ExternalParam struct {
	// Externally passed parameters that will not change
	ExParamConst
	// Inner parameters
	ExParam
}

Use for External table scan param

type FilterParam

type FilterParam struct {
	File2Size  map[string]int64
	FilterExpr *plan.Expr
	// contains filtered or unexported fields
}

type ParseLineHandler

type ParseLineHandler struct {
	// contains filtered or unexported fields
}

func GetSimdcsvReader

func GetSimdcsvReader(param *ExternalParam, proc *process.Process) (*ParseLineHandler, error)

GetSimdcsvReader get file reader from external file

type ZonemapFileparam

type ZonemapFileparam struct {
	// contains filtered or unexported fields
}