lexer

Documentation

Index

• func GenLexer(lexSpec *spec.LexicalSpec, pkgName string) ([]byte, error)
• func NewLexSpec(spec *spec.LexicalSpec) *lexSpec
• type KindID
  • func (id KindID) Int() int
• type LexSpec
• type Lexer
  • func NewLexer(spec LexSpec, src io.Reader, opts ...LexerOption) (*Lexer, error)
  • func (l *Lexer) Mode() ModeID
  • func (l *Lexer) Next() (*Token, error)
  • func (l *Lexer) PopMode() error
  • func (l *Lexer) PushMode(mode ModeID)
• type LexerOption
  • func DisableModeTransition() LexerOption
• type ModeID
  • func (id ModeID) Int() int
• type ModeKindID
  • func (id ModeKindID) Int() int
• type StateID
  • func (id StateID) Int() int
• type Token

Functions

func GenLexer

func GenLexer(lexSpec *spec.LexicalSpec, pkgName string) ([]byte, error)

func NewLexSpec

func NewLexSpec(spec *spec.LexicalSpec) *lexSpec

Types

type KindID

type KindID int

func (KindID) Int

func (id KindID) Int() int

type LexSpec

type LexSpec interface {
	InitialMode() ModeID
	Pop(mode ModeID, modeKind ModeKindID) bool
	Push(mode ModeID, modeKind ModeKindID) (ModeID, bool)
	ModeName(mode ModeID) string
	InitialState(mode ModeID) StateID
	NextState(mode ModeID, state StateID, v int) (StateID, bool)
	Accept(mode ModeID, state StateID) (ModeKindID, bool)
	KindIDAndName(mode ModeID, modeKind ModeKindID) (KindID, string)
}

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(spec LexSpec, src io.Reader, opts ...LexerOption) (*Lexer, error)

NewLexer returns a new lexer.

func (*Lexer) Mode

func (l *Lexer) Mode() ModeID

Mode returns the current lex mode.

func (*Lexer) Next

func (l *Lexer) Next() (*Token, error)

Next returns a next token.

func (*Lexer) PopMode

func (l *Lexer) PopMode() error

PopMode removes a lex mode from the top of the mode stack.

func (*Lexer) PushMode

func (l *Lexer) PushMode(mode ModeID)

PushMode adds a lex mode onto the mode stack.

type LexerOption

type LexerOption func(l *Lexer) error

func DisableModeTransition

func DisableModeTransition() LexerOption

DisableModeTransition disables the active mode transition. Thus, even if the lexical specification has the push and pop operations, the lexer doesn't perform these operations. When the lexical specification has multiple modes, and this option is enabled, you need to call the Lexer.Push and Lexer.Pop methods to perform the mode transition. You can use the Lexer.Mode method to know the current lex mode.

type ModeID

type ModeID int

func (ModeID) Int

func (id ModeID) Int() int

type ModeKindID

type ModeKindID int

func (ModeKindID) Int

func (id ModeKindID) Int() int

type StateID

type StateID int

func (StateID) Int

func (id StateID) Int() int

type Token

type Token struct {
	// ModeID is an ID of a lex mode.
	ModeID ModeID

	// KindID is an ID of a kind. This is unique among all modes.
	KindID KindID

	// ModeKindID is an ID of a lexical kind. This is unique only within a mode.
	// Note that you need to use KindID field if you want to identify a kind across all modes.
	ModeKindID ModeKindID

	// Row is a row number where a lexeme appears.
	Row int

	// Col is a column number where a lexeme appears.
	// Note that Col is counted in code points, not bytes.
	Col int

	// Lexeme is a byte sequence matched a pattern of a lexical specification.
	Lexeme []byte

	// When this field is true, it means the token is the EOF token.
	EOF bool

	// When this field is true, it means the token is an error token.
	Invalid bool
}

Token representes a token.