tokenizer

Documentation

Index

• func IsEqual(a rune) matchFn
• func IsNumeric(i rune) bool
• func IsOneOf(a ...rune) matchFn
• type StateFn
• type Tokenizer
  • func NewTokenizer[Token any](input io.RuneScanner, initialState StateFn[Token]) *Tokenizer[Token]
  • func (t *Tokenizer[Token]) Done() bool
  • func (t *Tokenizer[Token]) Ignore()
  • func (t *Tokenizer[Token]) IgnoreUntil(fn matchFn)
  • func (t *Tokenizer[Token]) IgnoreWhile(fn matchFn)
  • func (t *Tokenizer[Token]) NextRune() rune
  • func (t *Tokenizer[Token]) Peek() rune
  • func (t *Tokenizer[Token]) PeekMatch(fn matchFn) bool
  • func (t *Tokenizer[Token]) ReadUntil(fn matchFn) []rune
  • func (t *Tokenizer[Token]) Tokens() iter.Seq[Token]

Functions

func IsEqual

func IsEqual(a rune) matchFn

func IsNumeric

func IsNumeric(i rune) bool

func IsOneOf

func IsOneOf(a ...rune) matchFn

Types

type StateFn

type StateFn[Token any] func(t *Tokenizer[Token]) (*Token, StateFn[Token])

type Tokenizer

type Tokenizer[Token any] struct {
	// contains filtered or unexported fields
}

func NewTokenizer

func NewTokenizer[Token any](input io.RuneScanner, initialState StateFn[Token]) *Tokenizer[Token]

func (*Tokenizer[Token]) Done

func (t *Tokenizer[Token]) Done() bool

func (*Tokenizer[Token]) Ignore

func (t *Tokenizer[Token]) Ignore()

func (*Tokenizer[Token]) IgnoreUntil

func (t *Tokenizer[Token]) IgnoreUntil(fn matchFn)

func (*Tokenizer[Token]) IgnoreWhile

func (t *Tokenizer[Token]) IgnoreWhile(fn matchFn)

func (*Tokenizer[Token]) NextRune

func (t *Tokenizer[Token]) NextRune() rune

func (*Tokenizer[Token]) Peek

func (t *Tokenizer[Token]) Peek() rune

func (*Tokenizer[Token]) PeekMatch

func (t *Tokenizer[Token]) PeekMatch(fn matchFn) bool

func (*Tokenizer[Token]) ReadUntil

func (t *Tokenizer[Token]) ReadUntil(fn matchFn) []rune

func (*Tokenizer[Token]) Tokens

func (t *Tokenizer[Token]) Tokens() iter.Seq[Token]