lexer

Documentation

Overview

Package lexer provides a lexical analyser for the language. The lexer passes the tokens to a channel from which a client can read.

l := lexer.Lex(filename, reader)
for token := range l.Tokens {
	// do something with the token
	if token.Type == lexer.EOF {
		break // we are done
	}
}

Index

• type Lexer
  • func Lex(filename string, src io.Reader) *Lexer
• type Token
  • func (t Token) String() string
• type Type
  • func (i Type) String() string

Types

type Lexer

type Lexer struct {
	Tokens chan Token // channel of tokens
	// contains filtered or unexported fields
}

Lexer represents the lexical analyser.

func Lex

func Lex(filename string, src io.Reader) *Lexer

Lex creates a new lexer for the input source and starts analyzing the input source.

type Token

type Token struct {
	Pos  scanner.Position // position in the input string
	Text string           // text of this token
	Type Type             // tpye of this token
}

Token represents a token.

func (Token) String

func (t Token) String() string

type Type

type Type int

Type categorizes a token.

const (
	EOF   Type = iota // end of file
	Error             // error, value is the text of the lexeme
	Ident             // alphanumeric identifier
	// Literals
	False  // false
	Number // integer number
	True   // true
	// Keywords
	Else  // else
	For   // for
	If    // if
	Print // print
	Var   // var
	// Types
	Bool // bool
	Int  // int
	// Assignment operator
	Assign // =
	// Arithmetic operators
	Multiply // *
	Divide   // /
	Plus     // +
	Minus    // -
	// Relationship operators
	Less           // <
	LessOrEqual    // <=
	Greater        // >=
	GreaterOrEqual // >
	Equal          // ==
	NotEqual       // !=
	// Logical operators
	Not // !
	And // &&
	Or  // ||
	// Delimiters
	LeftParen  // (
	RightParen // )
	LeftBrace  // {
	RightBrace // }
)

func (Type) String

func (i Type) String() string