tokenizer

Documentation

Index

• Constants
• func DumpTokens(tokens <-chan Token, output io.Writer) error
• func DumpTokensFromFile(filename string, output io.Writer) error
• func FileToRuneChannel(filename string, output chan<- RuneLoc) error
• func IsValidInIdentifier(r rune) bool
• func ProcessIndentation(input <-chan Token, output chan<- Token) error
• func Tokenize(input <-chan RuneLoc, output chan<- Token) error
• type RuneLoc
• type SourceLocation
  • func SourceHere() SourceLocation
  • func (s SourceLocation) String() string
• type Token
  • func NoToken() Token
  • func (t Token) IsNone() bool
  • func (t Token) String() string
• type TokenType
  • func (t TokenType) String() string

Constants

const BeginTokens = "[begining of token stream]"

const EndTokens = "[end of token stream]"

const NoChar rune = 0

Functions

func DumpTokens

func DumpTokens(tokens <-chan Token, output io.Writer) error

func DumpTokensFromFile

func DumpTokensFromFile(filename string, output io.Writer) error

func FileToRuneChannel

func FileToRuneChannel(filename string, output chan<- RuneLoc) error

func IsValidInIdentifier

func IsValidInIdentifier(r rune) bool

func ProcessIndentation

func ProcessIndentation(input <-chan Token, output chan<- Token) error

func Tokenize

func Tokenize(input <-chan RuneLoc, output chan<- Token) error

Types

type RuneLoc

type RuneLoc struct {
	Rune rune
	Loc  SourceLocation
}

type SourceLocation

type SourceLocation struct {
	File   string
	Line   int
	Column int
}

func SourceHere

func SourceHere() SourceLocation

Used when injecting new code

func (SourceLocation) String

func (s SourceLocation) String() string

type Token

type Token struct {
	TokenType

	Int int64
	Str string
	Loc SourceLocation
	// contains filtered or unexported fields
}

func NoToken

func NoToken() Token

func (Token) IsNone

func (t Token) IsNone() bool

func (Token) String

func (t Token) String() string

type TokenType

type TokenType uint8

const (
	// we don't actually need TokNone; it's just useful to make sure that none of our tokens are equal to 0
	TokNone TokenType = iota

	// symbols
	TokSlash
	TokMinus
	TokSetEqual
	TokParenOpen
	TokParenClose
	TokComma
	TokDot
	TokDotDot
	TokColon
	TokSemicolon
	TokEquals
	TokNotEquals
	TokLessThan
	TokGreaterThan
	TokLessThanOrEquals
	TokGreaterThanOrEquals
	TokLeftShift
	TokRightShift
	TokNot

	// keywords
	TokKeywordIf
	TokKeywordReturn
	TokKeywordSet
	TokKeywordIn
	TokKeywordNew
	TokKeywordDel
	TokKeywordFor
	TokKeywordAs
	TokKeywordVar
	TokKeywordProc
	TokKeywordVerb
	TokPreprocessorDefine
	TokPreprocessorInclude

	// literals
	TokInteger
	TokSymbol
	TokResource
	TokStringStart
	TokStringEnd
	TokStringInsertStart
	TokStringInsertEnd
	TokStringLiteral

	// spacing
	TokNewline
	TokSpaces
	TokTabs
	TokIndent
	TokUnindent
)

func (TokenType) String

func (t TokenType) String() string