lexer

Documentation

Index

• Constants
• func Lex(ctx context.Context, input string) chan Token
• type Space
• type Token
  • func (t Token) Is(ty TokenType) bool
  • func (t Token) IsString(ty TokenType, s string) bool
• type TokenType
  • func (t TokenType) String() string

Constants

const EOF rune = -1

Functions

func Lex

func Lex(ctx context.Context, input string) chan Token

Types

type Space

type Space struct {
	String          string
	ContainsNewline bool
}

type Token

type Token struct {
	Type  TokenType
	Value interface{}
	ast.Range
}

func (Token) Is

func (t Token) Is(ty TokenType) bool

func (Token) IsString

func (t Token) IsString(ty TokenType, s string) bool

type TokenType

type TokenType uint8

const (
	TokenError TokenType = iota
	TokenEOF
	TokenSpace
	TokenBinaryLiteral
	TokenOctalLiteral
	TokenDecimalLiteral
	TokenHexadecimalLiteral
	TokenFixedPointLiteral
	TokenIdentifier
	TokenString
	TokenPlus
	TokenMinus
	TokenStar
	TokenSlash
	TokenPercent
	TokenDoubleQuestionMark
	TokenParenOpen
	TokenParenClose
	TokenBraceOpen
	TokenBraceClose
	TokenBracketOpen
	TokenBracketClose
	TokenQuestionMark
	TokenQuestionMarkDot
	TokenComma
	TokenColon
	TokenDot
	TokenSemicolon
	TokenLeftArrow
	TokenLeftArrowExclamation
	TokenSwap
	TokenLess
	TokenLessEqual
	TokenLessLess
	TokenGreater
	TokenGreaterEqual
	TokenGreaterGreater
	TokenEqual
	TokenEqualEqual
	TokenExclamationMark
	TokenNotEqual
	TokenBlockCommentStart
	TokenBlockCommentEnd
	TokenBlockCommentContent
	TokenLineComment
	TokenAmpersand
	TokenAmpersandAmpersand
	TokenCaret
	TokenVerticalBar
	TokenVerticalBarVerticalBar
	TokenAt
	TokenAsExclamationMark
	TokenAsQuestionMark
	// NOTE: not an actual token, must be last item
	TokenMax
)

func (TokenType) String

func (t TokenType) String() string