tokenizer

Documentation

Index

• Constants
• Variables
• type JackTokenizer
  • func NewJackTokenizer(sourceCode string) *JackTokenizer
  • func (jT *JackTokenizer) Advance()
  • func (jT *JackTokenizer) HasMoreTokens() bool
  • func (jT *JackTokenizer) Identifier() string
  • func (jT *JackTokenizer) IntVal() int
  • func (jT *JackTokenizer) KeyWord() string
  • func (jT *JackTokenizer) StringVal() string
  • func (jT *JackTokenizer) Symbol() uint8
  • func (jT *JackTokenizer) TokenType() TokenType
• type TokenType
  • func (tt TokenType) String() string
• type Tokenizer

Constants

const Quote = '"'

Variables

var Keywords = map[string]bool{
	"class":       true,
	"constructor": true,
	"function":    true,
	"method":      true,
	"field":       true,
	"static":      true,
	"var":         true,
	"int":         true,
	"char":        true,
	"boolean":     true,
	"void":        true,
	"true":        true,
	"false":       true,
	"null":        true,
	"this":        true,
	"let":         true,
	"do":          true,
	"if":          true,
	"else":        true,
	"while":       true,
	"return":      true,
}

var Symbols = map[uint8]bool{
	'{': true,
	'}': true,
	'(': true,
	')': true,
	'[': true,
	']': true,
	'.': true,
	',': true,
	';': true,
	'+': true,
	'-': true,
	'*': true,
	'/': true,
	'&': true,
	'|': true,
	'<': true,
	'>': true,
	'=': true,
	'~': true,
}

Types

type JackTokenizer

type JackTokenizer struct {
	// contains filtered or unexported fields
}

func NewJackTokenizer

func NewJackTokenizer(sourceCode string) *JackTokenizer

func (*JackTokenizer) Advance

func (jT *JackTokenizer) Advance()

func (*JackTokenizer) HasMoreTokens

func (jT *JackTokenizer) HasMoreTokens() bool

func (*JackTokenizer) Identifier

func (jT *JackTokenizer) Identifier() string

func (*JackTokenizer) IntVal

func (jT *JackTokenizer) IntVal() int

func (*JackTokenizer) KeyWord

func (jT *JackTokenizer) KeyWord() string

func (*JackTokenizer) StringVal

func (jT *JackTokenizer) StringVal() string

func (*JackTokenizer) Symbol

func (jT *JackTokenizer) Symbol() uint8

func (*JackTokenizer) TokenType

func (jT *JackTokenizer) TokenType() TokenType

type TokenType

type TokenType int

const (
	Keyword TokenType = iota
	Symbol
	Identifier
	IntConst
	StringConst
	UnknownToken
)

func (TokenType) String

func (tt TokenType) String() string

type Tokenizer

type Tokenizer interface {
	Advance()
	HasMoreTokens() bool
	TokenType() TokenType
	KeyWord() string
	Symbol() uint8
	Identifier() string
	IntVal() int
	StringVal() string
}