Versions in this module Expand all Collapse all v0 v0.0.2 May 27, 2024 Changes in this version + func ScanNext(input []rune, k TokenKind) (token []rune, err error) v0.0.1 Nov 13, 2023 Changes in this version + func IsDecimalDigit(r rune) bool + func IsKeyword(r []rune) bool + type ScanState struct + Cursor int + Input []rune + func (s ScanState) CountWhile(begin int, satisfy func(rune) bool) int + func (s ScanState) FindFirst(begin int, patternSize int, pattern func([]rune) bool) (int, bool) + func (s ScanState) Len() int + func (s ScanState) PeekAt(offset int) rune + func (s ScanState) PeekSlice(begin int, endExclusive int) []rune + type Token struct + Begin int + Column int + Content []rune + End int + Kind TokenKind + Line int + func Tokenize(input []rune) ([]Token, error) + func (t Token) IsValid() bool + type TokenKind int + const TokenComment + const TokenEOF + const TokenIdentifier + const TokenIdentifierQuoted + const TokenKeyword + const TokenLiteralFloat + const TokenLiteralInteger + const TokenLiteralQuoted + const TokenSpace + const TokenSpecialChar + const TokenUnspecified + func Comment(s *ScanState) (int, TokenKind, error) + func IdentifierOrKeyword(s *ScanState) (int, TokenKind, error) + func IdentifierQuoted(s *ScanState) (int, TokenKind, error) + func LiteralQuoted(s *ScanState) (int, TokenKind, error) + func NumberOrDot(s *ScanState) (int, TokenKind, error) + func Spaces(s *ScanState) (int, TokenKind, error) + func SpecialChar(s *ScanState) (int, TokenKind, error) + func (i TokenKind) String() string + type TokenScanner struct + func (s *TokenScanner) Init(input []rune) + func (s *TokenScanner) ScanNext() (Token, error)