lexer

Documentation

Index

• type Lexer
  • func NewLexer(input []rune, filename string) *Lexer
  • func (l *Lexer) NewToken(tokenType TokenType, literal string) Token
  • func (l *Lexer) NextToken() Token
  • func (l *Lexer) ReadChar() bool
• type Token
  • func (t Token) String() string
• type TokenType
  • func LookupKeyword(s string) TokenType
  • func (t TokenType) String() string

Types

type Lexer

type Lexer struct {
	Filename string
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(input []rune, filename string) *Lexer

NewLexer creates a new Lexer instance

func (*Lexer) NewToken

func (l *Lexer) NewToken(tokenType TokenType, literal string) Token

func (*Lexer) NextToken

func (l *Lexer) NextToken() Token

func (*Lexer) ReadChar

func (l *Lexer) ReadChar() bool

type Token

type Token struct {
	Type    TokenType
	Literal string

	File string
	Line int
	Col  int
}

func (Token) String

func (t Token) String() string

type TokenType

type TokenType int

const (
	TOKEN_UNKNOWN TokenType = iota
	TOKEN_EOF
	TOKEN_RESERVED
	// Keywords
	TOKEN_STRUCT
	TOKEN_ENUM
	TOKEN_ALIAS
	// Symbols
	TOKEN_OPEN_BRACE
	TOKEN_CLOSE_BRACE
	// Separators
	TOKEN_COMMA
	TOKEN_SEMICOLON
	// Operators
	TOKEN_EQUAL
	// Literals
	TOKEN_IDENTIFIER

	// built-in types
	TOKEN_RAWTYPE
)

func LookupKeyword

func LookupKeyword(s string) TokenType

func (TokenType) String

func (t TokenType) String() string