lex

Documentation

Index

• Constants
• func HasUChars(r rune, l *Lexer) bool
• func LexIRIRef(l *Lexer, styp ItemType) error
• type CheckRune
• type CheckRuneRec
• type ItemIterator
  • func (p *ItemIterator) Item() item
  • func (p *ItemIterator) Next() bool
  • func (p *ItemIterator) Peek(num int) ([]item, error)
  • func (p *ItemIterator) Prev() bool
• type ItemType
• type Lexer
  • func NewLexer(input string) *Lexer
  • func (l *Lexer) AcceptRun(c CheckRune) (lastr rune, validr bool)
  • func (l *Lexer) AcceptRunRec(c CheckRuneRec)
  • func (l *Lexer) AcceptRunTimes(c CheckRune, times int) int
  • func (l *Lexer) AcceptUntil(c CheckRune)
  • func (l *Lexer) Backup()
  • func (l *Lexer) Emit(t ItemType)
  • func (l *Lexer) Errorf(format string, args ...interface{}) StateFn
  • func (l *Lexer) Ignore()
  • func (l *Lexer) IgnoreRun(c CheckRune)
  • func (l *Lexer) NewIterator() *ItemIterator
  • func (l *Lexer) Next() (result rune)
  • func (l *Lexer) Peek() rune
  • func (l *Lexer) Run(f StateFn) *Lexer
• type StateFn

Constants

const EOF = -1

Functions

func HasUChars

func HasUChars(r rune, l *Lexer) bool

UCHAR ::= '\u' HEX HEX HEX HEX | '\U' HEX HEX HEX HEX HEX HEX HEX HEX

func LexIRIRef

func LexIRIRef(l *Lexer, styp ItemType) error

Types

type CheckRune

type CheckRune func(r rune) bool

CheckRune is predicate signature for accepting valid runes on input.

type CheckRuneRec

type CheckRuneRec func(r rune, l *Lexer) bool

CheckRuneRec is like CheckRune with Lexer as extra argument. This can be used to recursively call other CheckRune(s).

type ItemIterator

type ItemIterator struct {
	// contains filtered or unexported fields
}

func (*ItemIterator) Item

func (p *ItemIterator) Item() item

Item returns the current item and advances the index.

func (*ItemIterator) Next

func (p *ItemIterator) Next() bool

Valid returns true if we haven't consumed all the items.

func (*ItemIterator) Peek

func (p *ItemIterator) Peek(num int) ([]item, error)

Peek returns the next n items without consuming them.

func (*ItemIterator) Prev

func (p *ItemIterator) Prev() bool

Prev moves the index back by one.

type ItemType

type ItemType int

ItemType is used to set the type of a token. These constants can be defined in the file containing state functions. Note that their value should be >= 5.

const (
	ItemEOF   ItemType = iota
	ItemError          // error
)

type Lexer

type Lexer struct {
	// NOTE: Using a text scanner wouldn't work because it's designed for parsing
	// Golang. It won't keep track of Start Position, or allow us to retrieve
	// slice from [Start:Pos]. Better to just use normal string.
	Input string // string being scanned.
	Start int    // Start Position of this item.
	Pos   int    // current Position of this item.
	Width int    // Width of last rune read from input.

	Depth           int  // nesting of {}
	ArgDepth        int  // nesting of ()
	Mode            int  // mode based on information so far.
	InsideDirective bool // To indicate we are inside directive.
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(input string) *Lexer

func (*Lexer) AcceptRun

func (l *Lexer) AcceptRun(c CheckRune) (lastr rune, validr bool)

AcceptRun accepts tokens based on CheckRune untill it returns false or EOF is reached. Returns last rune accepted and valid flag for rune.

func (*Lexer) AcceptRunRec

func (l *Lexer) AcceptRunRec(c CheckRuneRec)

AcceptRunRec accepts tokens based on CheckRuneRec untill it returns false or EOF is reached.

func (*Lexer) AcceptRunTimes

func (l *Lexer) AcceptRunTimes(c CheckRune, times int) int

AcceptRunTimes accepts tokens with CheckRune given number of times. returns number of times it was successful.

func (*Lexer) AcceptUntil

func (l *Lexer) AcceptUntil(c CheckRune)

AcceptUntil accepts tokens based on CheckRune till it returns false or EOF is reached.

func (*Lexer) Backup

func (l *Lexer) Backup()

func (*Lexer) Emit

func (l *Lexer) Emit(t ItemType)

Emit emits the item with it's type information.

func (*Lexer) Errorf

func (l *Lexer) Errorf(format string, args ...interface{}) StateFn

Errorf returns the error state function.

func (*Lexer) Ignore

func (l *Lexer) Ignore()

func (*Lexer) IgnoreRun

func (l *Lexer) IgnoreRun(c CheckRune)

func (*Lexer) NewIterator

func (l *Lexer) NewIterator() *ItemIterator

func (*Lexer) Next

func (l *Lexer) Next() (result rune)

Next reads the next rune from the Input, sets the Width and advances Pos.

func (*Lexer) Peek

func (l *Lexer) Peek() rune

func (*Lexer) Run

func (l *Lexer) Run(f StateFn) *Lexer

type StateFn

type StateFn func(*Lexer) StateFn

stateFn represents the state of the scanner as a function that returns the next state.