token

Documentation

Index

• func ComparePos(x, y Pos) int
• type Kind
  • func (i Kind) String() string
• type Pos
  • func NewPos(line, col int) Pos
  • func (p *Pos) String() string
• type SQLWord
  • func MakeKeyword(word string, quoteStyle rune) *SQLWord
  • func (s *SQLWord) NoQuoteString() string
  • func (s *SQLWord) String() string
• type Token
• type Tokenizer
  • func NewTokenizer(src io.Reader, dialect dialect.Dialect) *Tokenizer
  • func (t *Tokenizer) NextToken() (*Token, error)
  • func (t *Tokenizer) Pos() Pos
  • func (t *Tokenizer) Tokenize() ([]*Token, error)

Functions

func ComparePos

func ComparePos(x, y Pos) int

Types

type Kind

type Kind int

const (
	// A keyword (like SELECT)
	SQLKeyword Kind = iota
	// Numeric literal
	Number
	// A character that cloud not be tokenized
	Char
	// Single quoted string i.e: 'string'
	SingleQuotedString
	// National string i.e: N'string'
	NationalStringLiteral
	// Comma
	Comma
	// Whitespace
	Whitespace
	// comment node
	Comment
	// multiline comment node
	MultilineComment
	// = operator
	Eq
	// != or <> operator
	Neq
	// <  operator
	Lt
	// > operator
	Gt
	// <= operator
	LtEq
	// >= operator
	GtEq
	// + operator
	Plus
	// - operator
	Minus
	// * operator
	Mult
	// / operator
	Div
	// % operator
	Caret
	// ^ operator
	Mod
	// Left parenthesis `(`
	LParen
	// Right parenthesis `)`
	RParen
	// Period
	Period
	// Colon
	Colon
	// DoubleColon
	DoubleColon
	// Semicolon
	Semicolon
	// Backslash
	Backslash
	// Left bracket `]`
	LBracket
	// Right bracket `[`
	RBracket
	// &
	Ampersand
	// Left brace `{`
	LBrace
	// Right brace `}`
	RBrace
	// ILLEGAL sqltoken
	ILLEGAL
)

func (Kind) String

func (i Kind) String() string

type Pos

type Pos struct {
	Line int
	Col  int
}

func NewPos

func NewPos(line, col int) Pos

func (*Pos) String

func (p *Pos) String() string

type SQLWord

type SQLWord struct {
	Value      string
	QuoteStyle rune
	Keyword    string
	Kind       dialect.KeywordKind
}

func MakeKeyword

func MakeKeyword(word string, quoteStyle rune) *SQLWord

func (*SQLWord) NoQuoteString

func (s *SQLWord) NoQuoteString() string

func (*SQLWord) String

func (s *SQLWord) String() string

type Token

type Token struct {
	Kind  Kind
	Value interface{}
	From  Pos
	To    Pos
}

type Tokenizer

type Tokenizer struct {
	Dialect dialect.Dialect
	Scanner *scanner.Scanner
	Line    int
	Col     int
}

func NewTokenizer

func NewTokenizer(src io.Reader, dialect dialect.Dialect) *Tokenizer

func (*Tokenizer) NextToken

func (t *Tokenizer) NextToken() (*Token, error)

func (*Tokenizer) Pos

func (t *Tokenizer) Pos() Pos

func (*Tokenizer) Tokenize

func (t *Tokenizer) Tokenize() ([]*Token, error)