lexer

Documentation

Index

• Constants
• type Space
• type Token
  • func (t Token) Is(ty TokenType) bool
  • func (t Token) IsString(ty TokenType, s string) bool
• type TokenLimitReachedError
  • func (t TokenLimitReachedError) Error() string
• type TokenStream
  • func Lex(input string, memoryGauge common.MemoryGauge) TokenStream
• type TokenType
  • func (t TokenType) IsIntegerLiteral() bool
  • func (t TokenType) String() string

Constants

const EOF rune = -1

Types

type Space

type Space struct {
	String          string
	ContainsNewline bool
}

type Token

type Token struct {
	Type  TokenType
	Value interface{}
	ast.Range
}

func (Token) Is

func (t Token) Is(ty TokenType) bool

func (Token) IsString

func (t Token) IsString(ty TokenType, s string) bool

type TokenLimitReachedError

type TokenLimitReachedError struct {
	ast.Position
}

func (TokenLimitReachedError) Error

func (t TokenLimitReachedError) Error() string

type TokenStream

type TokenStream interface {
	// Next consumes and returns one Token. If there are no tokens remaining, it returns Token{TokenEOF}
	Next() Token
	Cursor() int
	Revert(cursor int)
	// Input returns the whole input as source code
	Input() string
	Reclaim()
}

func Lex

func Lex(input string, memoryGauge common.MemoryGauge) TokenStream

type TokenType

type TokenType uint8

const (
	TokenError TokenType = iota
	TokenEOF
	TokenSpace
	TokenBinaryIntegerLiteral
	TokenOctalIntegerLiteral
	TokenDecimalIntegerLiteral
	TokenHexadecimalIntegerLiteral
	TokenUnknownBaseIntegerLiteral
	TokenFixedPointNumberLiteral
	TokenIdentifier
	TokenString
	TokenPlus
	TokenMinus
	TokenStar
	TokenSlash
	TokenPercent
	TokenDoubleQuestionMark
	TokenParenOpen
	TokenParenClose
	TokenBraceOpen
	TokenBraceClose
	TokenBracketOpen
	TokenBracketClose
	TokenQuestionMark
	TokenQuestionMarkDot
	TokenComma
	TokenColon
	TokenDot
	TokenSemicolon
	TokenLeftArrow
	TokenLeftArrowExclamation
	TokenSwap
	TokenLess
	TokenLessEqual
	TokenLessLess
	TokenGreater
	TokenGreaterEqual
	TokenEqual
	TokenEqualEqual
	TokenExclamationMark
	TokenNotEqual
	TokenBlockCommentStart
	TokenBlockCommentEnd
	TokenBlockCommentContent
	TokenLineComment
	TokenAmpersand
	TokenAmpersandAmpersand
	TokenCaret
	TokenVerticalBar
	TokenVerticalBarVerticalBar
	TokenAt
	TokenAsExclamationMark
	TokenAsQuestionMark
	TokenPragma
	// NOTE: not an actual token, must be last item
	TokenMax
)

func (TokenType) IsIntegerLiteral

func (t TokenType) IsIntegerLiteral() bool

func (TokenType) String

func (t TokenType) String() string