lex

Documentation

Index

• Constants
• func IsDigit(r rune) bool
• func IsIdent(r rune) bool
• func IsPlusMinus(r rune) bool
• func IsVariableStart(r rune) bool
• func IsVariableSuccessor(r rune) bool
• type Lexer
  • func (l *Lexer) Expect(expected string) error
  • func (l *Lexer) Next() *Token
  • func (l *Lexer) Parse(r io.Reader)
  • func (l *Lexer) Peek() *Token
  • func (l *Lexer) Skip() *Token
• type Token
  • func (t *Token) Clone(s string) *Token
  • func (t *Token) Text() string
  • func (t *Token) Token() rune

Constants

const (
	EOF       = scanner.EOF
	Ident     = scanner.Ident
	Int       = scanner.Int
	Float     = scanner.Float
	Char      = scanner.Char
	String    = scanner.String
	RawString = scanner.RawString
	Comment   = scanner.Comment
)

const (
	TOKEN_VARIABLE = -(iota + 100)
)

Functions

func IsDigit

func IsDigit(r rune) bool

func IsIdent

func IsIdent(r rune) bool

Returns true if a rune should be matched as an ident

func IsPlusMinus

func IsPlusMinus(r rune) bool

func IsVariableStart

func IsVariableStart(r rune) bool

IsVariableStart true if the rune is valid for the first char of a variable name

func IsVariableSuccessor

func IsVariableSuccessor(r rune) bool

IsVariableSuccessor true if the rune is valid for the successive chars in a variable name

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

A simple lexer used to tokenise raw text into something the Parser can handle

func (*Lexer) Expect

func (l *Lexer) Expect(expected string) error

Expect will return an error if the next token is not that expected

func (*Lexer) Next

func (l *Lexer) Next() *Token

Get the current token and move forward one place

func (*Lexer) Parse

func (l *Lexer) Parse(r io.Reader)

func (*Lexer) Peek

func (l *Lexer) Peek() *Token

Get the current token but do not move

func (*Lexer) Skip

func (l *Lexer) Skip() *Token

Skip skips the current token, same as Next() then Peek()

type Token

type Token struct {
	// contains filtered or unexported fields
}

A parsed token from the Lexer

func (*Token) Clone

func (t *Token) Clone(s string) *Token

Clone returns a new token with its text set to the new string. If the string is the same as the current text then the current token is returned.

func (*Token) Text

func (t *Token) Text() string

Text returns the text of this token

func (*Token) Token

func (t *Token) Token() rune

Token returns the rune for this token