lexer

Documentation

Index

• Constants
• Variables
• func IsAlphabet(c rune) bool
• func IsAlphabetOrDigit(c rune) bool
• func IsDigit(c rune) bool
• func IsWhiteSpace(c rune) bool
• type CharacterSet
  • func NewCharacterSet(alphabets, digits []rune) *CharacterSet
  • func NewCharacterSetWithDefault() *CharacterSet
  • func (cs *CharacterSet) GetAlphabets() []rune
  • func (cs *CharacterSet) GetDigits() []rune
• type DFA
  • func NewDFA(cs *CharacterSet) *DFA
  • func NewDFAWithDefault() *DFA
  • func (dfa *DFA) Match(runes []rune) *token.Token
  • func (dfa *DFA) Print()
• type Lexer
  • func NewLexer(fa dependency.Lexer) *Lexer
  • func (l *Lexer) GetFiniteAutomata() dependency.Lexer
  • func (l *Lexer) Lex(sql string) []*token.Token
• type NFA
  • func NewNFA(cs *CharacterSet) *NFA
  • func NewNFAWithDefault() *NFA
  • func (nfa *NFA) Match(runes []rune) *token.Token
  • func (nfa *NFA) Print()
• type Set
  • func NewSet(i int) *Set
  • func (s *Set) AddNext(c rune, ns *Set)
  • func (s *Set) AddState(state *State)
  • func (s *Set) Contains(state *State) bool
  • func (s *Set) Equal(other *Set) bool
  • func (s *Set) GetFinalState() (int, *State)
  • func (s *Set) Print()
  • func (s *Set) String() string
• type State
  • func NewState(i int) *State
  • func (s *State) AddNext(c rune, ns *State)
  • func (s *State) EpsilonMove() []*State
  • func (s *State) Print()

Constants

const (
	// keyword
	SelectString = "select"
	FromString   = "from"
	AsString     = "as"
	WhereString  = "where"
	AndString    = "and"
	OrString     = "or"
	// comparison operator
	GEString        = ">="
	LEString        = "<="
	NotEqual1String = "!="
	NotEqual2String = "<>"
)

const (
	// char
	UnderBarRune = '_'
	// comparison operator
	GTRune          = '>'
	LTRune          = '<'
	EqualRune       = '='
	ExclamationRune = '!'
	// arithmetic operator
	PlusRune     = '+'
	MinusRune    = '-'
	MultiplyRune = '*'
	DivideRune   = '/'
	ModRune      = '%'
	// separator
	CommaRune            = ','
	SemicolonRune        = ';'
	LeftParenthesisRune  = '('
	RightParenthesisRune = ')'
	SingleQuoteRune      = '\''
	// white space
	SpaceRune   = ' '
	TabRune     = '\t'
	ReturnRune  = '\r'
	NewLineRune = '\n'
)

const (
	EpsilonRune = 'ε'
)

Variables

var (
	MultiRuneMap = map[token.Type]string{

		token.Select: SelectString,
		token.From:   FromString,
		token.As:     AsString,
		token.Where:  WhereString,
		token.And:    AndString,
		token.Or:     OrString,

		token.GE:        GEString,
		token.LE:        LEString,
		token.NotEqual1: NotEqual1String,
		token.NotEqual2: NotEqual2String,
	}
	SingleRuneMap = map[token.Type]rune{

		token.GT:    GTRune,
		token.LT:    LTRune,
		token.Equal: EqualRune,

		token.Plus:     PlusRune,
		token.Minus:    MinusRune,
		token.Multiply: MultiplyRune,
		token.Divide:   DivideRune,
		token.Mod:      ModRune,

		token.LeftParenthesis:  LeftParenthesisRune,
		token.RightParenthesis: RightParenthesisRune,

		token.Comma:     CommaRune,
		token.Semicolon: SemicolonRune,
	}
)

Functions

func IsAlphabet

func IsAlphabet(c rune) bool

IsAlphabet returns if the given rune is an alphabet

func IsAlphabetOrDigit

func IsAlphabetOrDigit(c rune) bool

IsWhiteSpace returns if the given rune is either an alphabet or a digit

func IsDigit

func IsDigit(c rune) bool

IsDigit returns if the given rune is a digit

func IsWhiteSpace

func IsWhiteSpace(c rune) bool

IsWhiteSpace returns if the given rune is a white space

Types

type CharacterSet

type CharacterSet struct {
	Alphabets []rune
	Digits    []rune
}

func NewCharacterSet

func NewCharacterSet(alphabets, digits []rune) *CharacterSet

NewCharacterSet returns a new *CharacterSet

func NewCharacterSetWithDefault

func NewCharacterSetWithDefault() *CharacterSet

NewCharacterSetWithDefault returns a new *CharacterSet with default

func (*CharacterSet) GetAlphabets

func (cs *CharacterSet) GetAlphabets() []rune

GetAlphabets returns the alphabet runes

func (*CharacterSet) GetDigits

func (cs *CharacterSet) GetDigits() []rune

GetAlphabets returns the digit runes

type DFA

type DFA struct {
	CharacterSet *CharacterSet
	NFA          *NFA
	Index        int
	InitSet      *Set
}

func NewDFA

func NewDFA(cs *CharacterSet) *DFA

NewDFA returns a new *DFA

func NewDFAWithDefault

func NewDFAWithDefault() *DFA

NewDFAWithDefault returns a new *DFA with default

func (*DFA) Match

func (dfa *DFA) Match(runes []rune) *token.Token

Match matches the given runes and returns proper token

func (*DFA) Print

func (dfa *DFA) Print()

Print prints all the sets including the state in it

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(fa dependency.Lexer) *Lexer

NewLexer returns a new *Lexer

func (*Lexer) GetFiniteAutomata

func (l *Lexer) GetFiniteAutomata() dependency.Lexer

GetFiniteAutomata returns the finite automata of the lexer

func (*Lexer) Lex

func (l *Lexer) Lex(sql string) []*token.Token

Lex scans the input string and returns a token list

type NFA

type NFA struct {
	CharacterSet *CharacterSet
	Index        int
	InitState    *State
}

func NewNFA

func NewNFA(cs *CharacterSet) *NFA

NewNFA returns a new *NFA

func NewNFAWithDefault

func NewNFAWithDefault() *NFA

NewNFAWithDefault returns a new *NFA with default

func (*NFA) Match

func (nfa *NFA) Match(runes []rune) *token.Token

Match matches the given runes and returns proper token

func (*NFA) Print

func (nfa *NFA) Print()

Print prints all the states

type Set

type Set struct {
	States    []*State
	Index     int
	Next      map[rune]*Set
	IsFinal   bool
	TokenType token.Type
}

func NewSet

func NewSet(i int) *Set

NewSet returns a new *Set

func (*Set) AddNext

func (s *Set) AddNext(c rune, ns *Set)

AddNext adds the next set of given rune

func (*Set) AddState

func (s *Set) AddState(state *State)

AddState add the given state into the set,

• if a same state already exists in the set, it will be ignored
• if the new state is a final state and there is already a final state exists in the set, only one state of which token type is keyword stays in the set

func (*Set) Contains

func (s *Set) Contains(state *State) bool

Contains returns if the given state is in the set

func (*Set) Equal

func (s *Set) Equal(other *Set) bool

Equal returns if the twe sets contains the same states, the order of the states does not matter

func (*Set) GetFinalState

func (s *Set) GetFinalState() (int, *State)

GetFinalState returns the final state in the set

func (*Set) Print

func (s *Set) Print()

Print prints the set and all the next sets recursively

func (*Set) String

func (s *Set) String() string

String returns the string representation of the set

type State

type State struct {
	Index     int
	Next      map[rune][]*State
	IsFinal   bool
	TokenType token.Type
}

func NewState

func NewState(i int) *State

NewState returns a new *State

func (*State) AddNext

func (s *State) AddNext(c rune, ns *State)

AddNext adds the next state of the given rune

func (*State) EpsilonMove

func (s *State) EpsilonMove() []*State

EpsilonMove gets all the states that can transit to by epsilon move, it includes itself

func (*State) Print

func (s *State) Print()

Print prints the state and all the next states recursively