external

Documentation

Index

• Constants
• Variables
• func FilterFileList(ctx context.Context, node *plan.Node, proc *process.Process, fileList []string, ...) ([]string, []int64, error)
• func GetCompressType(param *tree.ExternParam, filepath string) string
• func ReadFileOffsetNoStrict(param *tree.ExternParam, mcpu int, fileSize int64) ([]int64, error)
• func ReadFileOffsetStrict(param *tree.ExternParam, mcpu int, fileSize int64, visibleCols []*plan.ColDef) ([]int64, error)
• type Argument
  • func NewArgument() *Argument
  • func (arg *Argument) Call(proc *process.Process) (vm.CallResult, error)
  • func (arg *Argument) Free(proc *process.Process, pipelineFailed bool, err error)
  • func (arg *Argument) GetOperatorBase() *vm.OperatorBase
  • func (arg *Argument) Prepare(proc *process.Process) error
  • func (arg *Argument) Release()
  • func (arg *Argument) String(buf *bytes.Buffer)
  • func (arg Argument) TypeName() string
  • func (arg *Argument) WithEs(es *ExternalParam) *Argument
• type ExFileparam
• type ExParam
• type ExParamConst
• type ExternalParam
• type FilterParam
• type ParquetHandler
• type ParseLineHandler
• type ZonemapFileparam

Constants

const (
	ColumnCntLargerErrorInfo = "the table column is larger than input data column"
)

const JsonNull = "\\N"

Variables

var (
	OneBatchMaxRow   = int(options.DefaultBlockMaxRows)
	S3ParallelMaxnum = 10
)

var (
	STATEMENT_ACCOUNT = "account"
)

Functions

func FilterFileList

func FilterFileList(ctx context.Context, node *plan.Node, proc *process.Process, fileList []string, fileSize []int64) ([]string, []int64, error)

func GetCompressType

func GetCompressType(param *tree.ExternParam, filepath string) string

func ReadFileOffsetNoStrict

func ReadFileOffsetNoStrict(param *tree.ExternParam, mcpu int, fileSize int64) ([]int64, error)

TODO : merge below two functions

func ReadFileOffsetStrict

func ReadFileOffsetStrict(param *tree.ExternParam, mcpu int, fileSize int64, visibleCols []*plan.ColDef) ([]int64, error)

Types

type Argument

type Argument struct {
	Es *ExternalParam

	vm.OperatorBase
	// contains filtered or unexported fields
}

func NewArgument

func NewArgument() *Argument

func (*Argument) Call

func (arg *Argument) Call(proc *process.Process) (vm.CallResult, error)

func (*Argument) Free

func (arg *Argument) Free(proc *process.Process, pipelineFailed bool, err error)

func (*Argument) GetOperatorBase

func (arg *Argument) GetOperatorBase() *vm.OperatorBase

func (*Argument) Prepare

func (arg *Argument) Prepare(proc *process.Process) error

func (*Argument) Release

func (arg *Argument) Release()

func (*Argument) String

func (arg *Argument) String(buf *bytes.Buffer)

func (Argument) TypeName

func (arg Argument) TypeName() string

func (*Argument) WithEs

func (arg *Argument) WithEs(es *ExternalParam) *Argument

type ExFileparam

type ExFileparam struct {
	End       bool
	FileCnt   int
	FileFin   int
	FileIndex int
	Filepath  string
}

type ExParam

type ExParam struct {
	Fileparam      *ExFileparam
	Zoneparam      *ZonemapFileparam
	Filter         *FilterParam
	MoCsvLineArray [][]csvparser.Field
	// contains filtered or unexported fields
}

type ExParamConst

type ExParamConst struct {
	IgnoreLine    int
	IgnoreLineTag int
	ParallelLoad  bool

	Idx             int
	CreateSql       string
	Close           byte
	Attrs           []string
	Cols            []*plan.ColDef
	FileList        []string
	FileSize        []int64
	FileOffset      []int64
	FileOffsetTotal []*pipeline.FileOffset
	Name2ColIndex   map[string]int32
	Ctx             context.Context
	Extern          *tree.ExternParam

	ClusterTable *plan.ClusterTable
	TblName      string // for debug moc3421
	// contains filtered or unexported fields
}

type ExternalParam

type ExternalParam struct {
	// Externally passed parameters that will not change
	ExParamConst
	// Inner parameters
	ExParam
}

Use for External table scan param

type FilterParam

type FilterParam struct {
	FilterExpr *plan.Expr
	// contains filtered or unexported fields
}

type ParquetHandler

type ParquetHandler struct {
	// contains filtered or unexported fields
}

type ParseLineHandler

type ParseLineHandler struct {
	// contains filtered or unexported fields
}

type ZonemapFileparam

type ZonemapFileparam struct {
	// contains filtered or unexported fields
}