lexer

Documentation

Index

• Constants
• type Space
• type Token
  • func (t Token) Is(ty TokenType) bool
  • func (t Token) Source(input []byte) []byte
• type TokenLimitReachedError
  • func (TokenLimitReachedError) Error() string
  • func (TokenLimitReachedError) IsUserError()
• type TokenStream
  • func Lex(input []byte, memoryGauge common.MemoryGauge) TokenStream
• type TokenType
  • func (t TokenType) IsIntegerLiteral() bool
  • func (t TokenType) String() string

Constants

const EOF rune = -1

Types

type Space

type Space struct {
	ContainsNewline bool
}

type Token

type Token struct {
	SpaceOrError any
	ast.Range
	Type TokenType
}

func (Token) Is

func (t Token) Is(ty TokenType) bool

func (Token) Source

func (t Token) Source(input []byte) []byte

type TokenLimitReachedError

type TokenLimitReachedError struct {
	ast.Position
}

func (TokenLimitReachedError) Error

func (TokenLimitReachedError) Error() string

func (TokenLimitReachedError) IsUserError

func (TokenLimitReachedError) IsUserError()

type TokenStream

type TokenStream interface {
	// Next consumes and returns one Token. If there are no tokens remaining, it returns Token{TokenEOF}
	Next() Token
	Cursor() int
	Revert(cursor int)
	// Input returns the whole input as source code
	Input() []byte
	Reclaim()
}

func Lex

func Lex(input []byte, memoryGauge common.MemoryGauge) TokenStream

type TokenType

type TokenType uint8

const (
	TokenError TokenType = iota
	TokenEOF
	TokenSpace
	TokenBinaryIntegerLiteral
	TokenOctalIntegerLiteral
	TokenDecimalIntegerLiteral
	TokenHexadecimalIntegerLiteral
	TokenUnknownBaseIntegerLiteral
	TokenFixedPointNumberLiteral
	TokenIdentifier
	TokenString
	TokenPlus
	TokenMinus
	TokenStar
	TokenSlash
	TokenPercent
	TokenDoubleQuestionMark
	TokenParenOpen
	TokenParenClose
	TokenBraceOpen
	TokenBraceClose
	TokenBracketOpen
	TokenBracketClose
	TokenQuestionMark
	TokenQuestionMarkDot
	TokenComma
	TokenColon
	TokenDot
	TokenSemicolon
	TokenLeftArrow
	TokenLeftArrowExclamation
	TokenSwap
	TokenLess
	TokenLessEqual
	TokenLessLess
	TokenGreater
	TokenGreaterEqual
	TokenEqual
	TokenEqualEqual
	TokenExclamationMark
	TokenNotEqual
	TokenBlockCommentStart
	TokenBlockCommentEnd
	TokenBlockCommentContent
	TokenLineComment
	TokenAmpersand
	TokenAmpersandAmpersand
	TokenCaret
	TokenVerticalBar
	TokenVerticalBarVerticalBar
	TokenAt
	TokenAsExclamationMark
	TokenAsQuestionMark
	TokenPragma
	// NOTE: not an actual token, must be last item
	TokenMax
)

func (TokenType) IsIntegerLiteral

func (t TokenType) IsIntegerLiteral() bool

func (TokenType) String

func (t TokenType) String() string