lexer

Documentation

Index

• type Lexer
  • func FromReader(input io.Reader, fileName string) *Lexer
  • func FromString(input string) *Lexer
  • func (lexer *Lexer) NextToken() token.Token
  • func (lexer *Lexer) Pos() scanner.Position
  • func (lexer *Lexer) RemainingTokens() ([]token.Token, error)

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

func FromReader

func FromReader(input io.Reader, fileName string) *Lexer

Creates a new Lexer from an input io.Reader, with the given file name

func FromString

func FromString(input string) *Lexer

Creates a new Lexer from an input string

func (*Lexer) NextToken

func (lexer *Lexer) NextToken() token.Token

Produces the next token from the lexer

func (*Lexer) Pos

func (lexer *Lexer) Pos() scanner.Position

Returns the current position of the scanner

func (*Lexer) RemainingTokens

func (lexer *Lexer) RemainingTokens() ([]token.Token, error)

Consumes the remaining tokens from the lexer until EOF is reached, and returns them in a slice