csvparser

Documentation

Index

• Variables
• func IndexAnyByte(s []byte, as *byteSet) int
• type CSVConfig
• type CSVParser
  • func NewCSVParser(cfg *CSVConfig, reader io.Reader, blockBufSize int64, shouldParseHeader bool, ...) (*CSVParser, error)
  • func (parser *CSVParser) Columns() []string
  • func (parser *CSVParser) Pos() int64
  • func (parser *CSVParser) Read() (row []Field, err error)
  • func (parser *CSVParser) SetColumns(columns []string)
• type Field

Variables

var (
	BufferSizeScale       = int64(5)
	ReadBlockSize   int64 = 64 * 1024
	// LargestEntryLimit is the max size for reading file to buf
	LargestEntryLimit = 10 * 1024 * 1024
)

Functions

func IndexAnyByte

func IndexAnyByte(s []byte, as *byteSet) int

IndexAnyByte returns the byte index of the first occurrence in s of any in the byte points in chars. It returns -1 if there is no code point in common.

Types

type CSVConfig

type CSVConfig struct {
	// they can only be used by LOAD DATA
	// https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-field-line-handling
	LinesStartingBy   string
	LinesTerminatedBy string

	FieldsTerminatedBy string
	FieldsEnclosedBy   string
	FieldsEscapedBy    string

	Null              []string
	Header            bool
	HeaderSchemaMatch bool
	TrimLastSep       bool
	NotNull           bool

	AllowEmptyLine bool
	// For non-empty FieldsEnclosedBy (for example quotes), null elements inside quotes are not considered as null except for
	// `\N` (when escape-by is `\`). That is to say, `\N` is special for null because it always means null.
	QuotedNullIsText bool
	// ref https://dev.mysql.com/doc/refman/8.0/en/load-data.html
	// > If the field begins with the ENCLOSED BY character, instances of that character are recognized as terminating a
	// > field value only if followed by the field or line TERMINATED BY sequence.
	// This means we will meet unescaped quote in a quoted field
	// > The "BIG" boss      -> The "BIG" boss
	// This means we will meet unescaped quote in an unquoted field
	UnescapedQuote bool

	// see csv.Reader
	Comment byte
}

type CSVParser

type CSVParser struct {
	// contains filtered or unexported fields
}

CSVParser is basically a copy of encoding/csv, but special-cased for MySQL-like input.

func NewCSVParser

func NewCSVParser(
	cfg *CSVConfig,
	reader io.Reader,
	blockBufSize int64,
	shouldParseHeader bool,
	reuseRow bool,
) (*CSVParser, error)

NewCSVParser creates a CSV parser.

func (*CSVParser) Columns

func (parser *CSVParser) Columns() []string

func (*CSVParser) Pos

func (parser *CSVParser) Pos() int64

func (*CSVParser) Read

func (parser *CSVParser) Read() (row []Field, err error)

func (*CSVParser) SetColumns

func (parser *CSVParser) SetColumns(columns []string)

type Field

type Field struct {
	Val    string
	IsNull bool
}