lex

Documentation

Index

• Constants
• func HasUChars(r rune, l *Lexer) bool
• func IsIRIChar(r rune, l *Lexer) bool
• func LexIRIRef(l *Lexer, styp ItemType) error
• type CheckRune
• type CheckRuneRec
• type Item
  • func (i Item) String() string
• type ItemIterator
  • func (p *ItemIterator) Item() Item
  • func (p *ItemIterator) Next() bool
  • func (p *ItemIterator) Peek(num int) ([]Item, error)
  • func (p *ItemIterator) PeekOne() (Item, bool)
  • func (p *ItemIterator) Prev() bool
  • func (p *ItemIterator) Restore(pos int)
  • func (p *ItemIterator) Save() int
• type ItemType
• type Lexer
  • func (l *Lexer) AcceptRun(c CheckRune) (lastr rune, validr bool)
  • func (l *Lexer) AcceptRunRec(c CheckRuneRec)
  • func (l *Lexer) AcceptRunTimes(c CheckRune, times int) int
  • func (l *Lexer) AcceptUntil(c CheckRune)
  • func (l *Lexer) Backup()
  • func (l *Lexer) Emit(t ItemType)
  • func (l *Lexer) Errorf(format string, args ...interface{}) StateFn
  • func (l *Lexer) Ignore()
  • func (l *Lexer) IgnoreRun(c CheckRune)
  • func (l *Lexer) IsEscChar(r rune) bool
  • func (l *Lexer) LexQuotedString() error
  • func (l *Lexer) NewIterator() *ItemIterator
  • func (l *Lexer) Next() (result rune)
  • func (l *Lexer) Peek() rune
  • func (l *Lexer) Run(f StateFn) *Lexer
• type StateFn

Constants

const EOF = -1

Functions

func HasUChars

func HasUChars(r rune, l *Lexer) bool

UCHAR ::= '\u' HEX HEX HEX HEX | '\U' HEX HEX HEX HEX HEX HEX HEX HEX

func IsIRIChar

func IsIRIChar(r rune, l *Lexer) bool

IRIREF ::= '<' ([^#x00-#x20<>"{}|^`\] | UCHAR)* '>'

func LexIRIRef

func LexIRIRef(l *Lexer, styp ItemType) error

Types

type CheckRune

type CheckRune func(r rune) bool

CheckRune is predicate signature for accepting valid runes on input.

type CheckRuneRec

type CheckRuneRec func(r rune, l *Lexer) bool

CheckRuneRec is like CheckRune with Lexer as extra argument. This can be used to recursively call other CheckRune(s).

type Item

type Item struct {
	Typ ItemType
	Val string
}

func (Item) String

func (i Item) String() string

type ItemIterator

type ItemIterator struct {
	// contains filtered or unexported fields
}

func (*ItemIterator) Item

func (p *ItemIterator) Item() Item

Item returns the current item.

func (*ItemIterator) Next

func (p *ItemIterator) Next() bool

Next advances the iterator by one.

func (*ItemIterator) Peek

func (p *ItemIterator) Peek(num int) ([]Item, error)

Peek returns the next n items without consuming them.

func (*ItemIterator) PeekOne

func (p *ItemIterator) PeekOne() (Item, bool)

PeekOne returns the next 1 item without consuming it.

func (*ItemIterator) Prev

func (p *ItemIterator) Prev() bool

Prev moves the index back by one.

func (*ItemIterator) Restore

func (p *ItemIterator) Restore(pos int)

Restore restores the the iterator to position specified.

func (*ItemIterator) Save

func (p *ItemIterator) Save() int

Save returns the current position of the iterator which we can use for restoring later.

type ItemType

type ItemType int

ItemType is used to set the type of a token. These constants can be defined in the file containing state functions. Note that their value should be >= 5.

const (
	ItemEOF   ItemType = iota
	ItemError          // error
)

type Lexer

type Lexer struct {
	// NOTE: Using a text scanner wouldn't work because it's designed for parsing
	// Golang. It won't keep track of Start Position, or allow us to retrieve
	// slice from [Start:Pos]. Better to just use normal string.
	Input string // string being scanned.
	Start int    // Start Position of this item.
	Pos   int    // current Position of this item.
	Width int    // Width of last rune read from input.

	Depth    int     // nesting of {}
	ArgDepth int     // nesting of ()
	Mode     StateFn // Default state to go back to after reading a token.
	// contains filtered or unexported fields
}

func (*Lexer) AcceptRun

func (l *Lexer) AcceptRun(c CheckRune) (lastr rune, validr bool)

AcceptRun accepts tokens based on CheckRune untill it returns false or EOF is reached. Returns last rune accepted and valid flag for rune.

func (*Lexer) AcceptRunRec

func (l *Lexer) AcceptRunRec(c CheckRuneRec)

AcceptRunRec accepts tokens based on CheckRuneRec untill it returns false or EOF is reached.

func (*Lexer) AcceptRunTimes

func (l *Lexer) AcceptRunTimes(c CheckRune, times int) int

AcceptRunTimes accepts tokens with CheckRune given number of times. returns number of times it was successful.

func (*Lexer) AcceptUntil

func (l *Lexer) AcceptUntil(c CheckRune)

AcceptUntil accepts tokens based on CheckRune till it returns false or EOF is reached.

func (*Lexer) Backup

func (l *Lexer) Backup()

func (*Lexer) Emit

func (l *Lexer) Emit(t ItemType)

Emit emits the item with it's type information.

func (*Lexer) Errorf

func (l *Lexer) Errorf(format string, args ...interface{}) StateFn

Errorf returns the error state function.

func (*Lexer) Ignore

func (l *Lexer) Ignore()

func (*Lexer) IgnoreRun

func (l *Lexer) IgnoreRun(c CheckRune)

func (*Lexer) IsEscChar

func (l *Lexer) IsEscChar(r rune) bool

ECHAR ::= '\' [tbnrf"'\]

func (*Lexer) LexQuotedString

func (l *Lexer) LexQuotedString() error

func (*Lexer) NewIterator

func (l *Lexer) NewIterator() *ItemIterator

func (*Lexer) Next

func (l *Lexer) Next() (result rune)

Next reads the next rune from the Input, sets the Width and advances Pos.

func (*Lexer) Peek

func (l *Lexer) Peek() rune

func (*Lexer) Run

func (l *Lexer) Run(f StateFn) *Lexer

type StateFn

type StateFn func(*Lexer) StateFn

stateFn represents the state of the scanner as a function that returns the next state.