lexer

Documentation

Index

• func PrintDebugTokens(tokens []token.Token)
• type Lexer
  • func NewLexer(source string) *Lexer
  • func NewLexerFromBinary(filepath string) *Lexer
  • func NewLexerFromSource(filepath string) *Lexer
  • func (l *Lexer) DebugTokensToNaive()
  • func (l *Lexer) DumpTokensToBinary(outputPath string)
  • func (l *Lexer) DumpTokensToFile(outputPath string)
  • func (l *Lexer) Tokenize() []token.Token

Functions

func PrintDebugTokens

func PrintDebugTokens(tokens []token.Token)

Types

type Lexer

type Lexer struct {
	CurrentLineNumber                  int
	CurrentLineExternalFileErrorReport int
	CurrentLineCharacterIndex          int

	Tokens      []token.Token
	TokenCursor int

	InputSource []rune
	InputCursor int

	TotalLinesProcessed int
}

func NewLexer

func NewLexer(source string) *Lexer

func NewLexerFromBinary

func NewLexerFromBinary(filepath string) *Lexer

func NewLexerFromSource

func NewLexerFromSource(filepath string) *Lexer

func (*Lexer) DebugTokensToNaive

func (l *Lexer) DebugTokensToNaive()

func (*Lexer) DumpTokensToBinary

func (l *Lexer) DumpTokensToBinary(outputPath string)

func (*Lexer) DumpTokensToFile

func (l *Lexer) DumpTokensToFile(outputPath string)

func (*Lexer) Tokenize

func (l *Lexer) Tokenize() []token.Token