lexer

Documentation

Index

• Constants
• func IsAlphaNumeric(r rune) bool
• func NextChar(input string) (rune, int)
• func TrimLeftSpace(input string) string
• type BaseToken
  • func (t *BaseToken) Type() TokenType
  • func (t *BaseToken) Value() string
• type Delim
• type FSMState
• type Lexer
  • func New(input string, sms FSMState) (Lexer, error)
• type StateHandler
• type Token
• type TokenType
• type Tokens

Constants

const EOF = -1

Functions

func IsAlphaNumeric

func IsAlphaNumeric(r rune) bool

IsAlphaNumeric reports whether r is an alphabetic, digit, or underscore.

func NextChar

func NextChar(input string) (rune, int)

func TrimLeftSpace

func TrimLeftSpace(input string) string

Types

type BaseToken

type BaseToken struct {
	Typ TokenType
	Val string
}

func (*BaseToken) Type

func (t *BaseToken) Type() TokenType

func (*BaseToken) Value

func (t *BaseToken) Value() string

type Delim

type Delim string

type FSMState

type FSMState interface {
	Init() fsm.State
	EoF() fsm.State
	Mapping() map[fsm.State]StateHandler
}

type Lexer

type Lexer interface {
	Next() Token
	Tokens() Tokens
	Emit(t Token)
}

func New

func New(input string, sms FSMState) (Lexer, error)

type StateHandler

type StateHandler func(lex Lexer) fsm.StateHandler

type Token

type Token interface {
	Type() TokenType
	Value() string
}

type TokenType

type TokenType int

const (
	TokenEOF TokenType = iota
)

type Tokens

type Tokens []Token