lex

Documentation

Index

• Constants
• Variables
• func IsAlphaNumeric(r rune) bool
• func IsEndOfLine(r rune) bool
• func IsSpace(r rune) bool
• func IsWhiteSpace(r rune) bool
• type BaseLexer
• type Lexer
• type Options
• type RunState
  • func (rs *RunState) Accept(valid string) bool
  • func (rs *RunState) AcceptRun(valid string) bool
  • func (rs *RunState) AtTerminator() bool
  • func (rs *RunState) Backup()
  • func (rs *RunState) Emit(t token.Typ)
  • func (rs *RunState) EmitToken(i *token.Token) StateFn
  • func (rs *RunState) Errorf(format string, args ...interface{}) *token.Token
  • func (rs *RunState) Ignore()
  • func (rs *RunState) Next() rune
  • func (rs *RunState) Peek() rune
  • func (rs *RunState) ScanNumber() bool
• type StateFn

Constants

const (
	RunePlus        = '+'
	RuneMinus       = '-'
	RuneAsterisk    = '*'
	RuneSlash       = '/'
	RunePercent     = '%'
	RuneComma       = ','
	RunePeriod      = '.'
	RuneExclamation = '!'
	RuneEqual       = '='
	RuneGreaterThan = '>'
	RuneLessThan    = '<'
	RuneSingleQuote = '\''
	RuneDoubleQuote = '"'
	RuneZero        = '0'
	RuneOne         = '1'
	RuneTwo         = '2'
	RuneThree       = '3'
	RuneFour        = '4'
	RuneFive        = '5'
	RuneSix         = '6'
	RuneSeven       = '7'
	RuneEight       = '8'
	RuneNine        = '9'
	RuneUnderscore  = '_'
	RuneLeftParen   = '('
	RuneRightParen  = ')'
	RuneSpace       = ' '
	RuneTab         = '\t'
	RuneCR          = '\r'
	RuneLF          = '\n'
	RuneA           = 'a'
	RuneB           = 'b'
	RuneC           = 'c'
	RuneD           = 'd'
	RuneE           = 'e'
	RuneF           = 'f'
	RuneG           = 'g'
	RuneH           = 'h'
	RuneI           = 'i'
	RuneJ           = 'j'
	RuneK           = 'k'
	RuneL           = 'l'
	RuneM           = 'm'
	RuneN           = 'n'
	RuneO           = 'o'
	RuneP           = 'p'
	RuneQ           = 'q'
	RuneR           = 'r'
	RuneS           = 's'
	RuneT           = 't'
	RuneU           = 'u'
	RuneV           = 'v'
	RuneW           = 'w'
	RuneX           = 'x'
	RuneY           = 'y'
	RuneZ           = 'z'
)

Common Runes. Using these ensures unicode dopplegangers aren't accidentally used

const EOF = -1

EOF represents EOF as a value of -1

Variables

var ErrBreak = errors.New("break")

ErrBreak prompts the loop to break immediately, but return a nil error to the caller and is not an actual fatal error (think EOF)

var ErrContinue = errors.New("continue")

ErrContinue prompts the loop to immediately continue to the next iteration and is not an actual fatal error (think EOF)

var ErrMissingRequiredKeyword = errors.New("missing required keyword")

ErrMissingRequiredKeyword indicates that the input's was missing a required keyword

var SpaceCharLookup = map[byte]interface{}{
	9:  nil,
	10: nil,
	13: nil,
	32: nil,
}

SpaceCharLookup is a map of acceptable space characters

Functions

func IsAlphaNumeric

func IsAlphaNumeric(r rune) bool

IsAlphaNumeric reports whether r is an alphabetic, digit.

func IsEndOfLine

func IsEndOfLine(r rune) bool

IsEndOfLine reports whether r is an end-of-line character.

func IsSpace

func IsSpace(r rune) bool

IsSpace reports whether r is a space character.

func IsWhiteSpace

func IsWhiteSpace(r rune) bool

IsWhiteSpace reports whether r is a whitespace character.

Types

type BaseLexer

type BaseLexer struct {
	Key map[string]token.Typ // the map of keywords to Typs to use when lexing
}

BaseLexer provides the parts needed to run the basic lexing framework

type Lexer

type Lexer interface {
	Run(string) token.Tokens
}

Lexer is the Lexer interface

type Options

type Options struct {
	CustomKeywords     map[string]token.Typ
	SpacedKeywordHints map[string]int
}

Options provides members that alter the behavior of the underlying Lexer

type RunState

type RunState struct {
	Input        string // the string being scanned
	InputLowered string // the lowercase version of the string being scanned
	InputWidth   int    // width of the input (to avoid multiple calls to len)
	Pos          int    // current position in the input
	Start        int    // start position of this Token
	Width        int    // width of last rune read from input
	ParenDepth   int    // nesting depth of ( ) exprs
	Tokens       token.Tokens
}

RunState contains all the information about a particular lexer run

func (*RunState) Accept

func (rs *RunState) Accept(valid string) bool

Accept consumes the next rune if it's from the valid set.

func (*RunState) AcceptRun

func (rs *RunState) AcceptRun(valid string) bool

AcceptRun consumes a run of runes from the valid set.

func (*RunState) AtTerminator

func (rs *RunState) AtTerminator() bool

AtTerminator reports whether the input is at valid termination character to appear after an identifier. Breaks .X.Y into two pieces. Also catches cases like "$x+2" not being acceptable without a space, in case we decide one day to implement arithmetic.

func (*RunState) Backup

func (rs *RunState) Backup()

Backup steps back one rune. Can only be called once per call of next.

func (*RunState) Emit

func (rs *RunState) Emit(t token.Typ)

Emit passes an Token back to the client.

func (*RunState) EmitToken

func (rs *RunState) EmitToken(i *token.Token) StateFn

EmitToken passes a pre-built an Token back to the client. Returning nil allows the function output to be used as a value when needd to consolidate EmitToken and the likely subsequent return nil

func (*RunState) Errorf

func (rs *RunState) Errorf(format string, args ...interface{}) *token.Token

Errorf returns an error token and terminates the scan by passing back a nil pointer that will be the next state, terminating l.NextToken.

func (*RunState) Ignore

func (rs *RunState) Ignore()

Ignore skips over the pending input before this point.

func (*RunState) Next

func (rs *RunState) Next() rune

Next returns the next rune in the input.

func (*RunState) Peek

func (rs *RunState) Peek() rune

Peek returns but does not consume the next rune in the input.

func (*RunState) ScanNumber

func (rs *RunState) ScanNumber() bool

ScanNumber returns true if pending input is a number

type StateFn

type StateFn func(Lexer, *RunState) StateFn

StateFn represents the state of the scanner as a function that returns the next state.