Documentation ¶
Index ¶
- Constants
- Variables
- func FormatTokenTable(ts []Token) string
- func Main(fn func(io.Reader) []Token)
- func MainWithArgs(fn func(io.Reader) []Token, args []string)
- func Repeat(fn func(), n int)
- func RunTests(t *testing.T, rules RuleSet, tests []TestCase)
- func While(s *Scanner, c Class, fn func())
- type Class
- type CommentRule
- type ExpRule
- type FracRule
- type IdentRule
- type IntRule
- type KindSet
- type NumRule
- type Pos
- type PrefixRule
- type Rule
- type RuleSet
- type Runner
- type Scanner
- func (s *Scanner) Discard()
- func (s *Scanner) Emit() Token
- func (s *Scanner) Illegal(format string, args ...any)
- func (s *Scanner) Init(src io.Reader)
- func (s *Scanner) InitFromBytes(src []byte)
- func (s *Scanner) InitFromString(src string)
- func (s *Scanner) Keep()
- func (s *Scanner) KeepIfPrefix(prefix string) bool
- func (s *Scanner) Lit() string
- func (s *Scanner) More() bool
- func (s *Scanner) Peek(n int) rune
- func (s *Scanner) PeekTo(n int) string
- func (s *Scanner) Skip()
- func (s *Scanner) Val() string
- func (s *Scanner) WriteRune(r rune)
- type StrRule
- type SuffixRule
- type SymbolRule
- type TestCase
- type Token
- type TokenError
Examples ¶
Constants ¶
View Source
const ( BinKind = "bin" CommentKind = "comment" ErrKind = "err" RealKind = "real" ExpKind = "exp" HexKind = "hex" IdentKind = "ident" IllegalKind = "illegal" IntKind = "int" NumKind = "num" OctKind = "oct" StrKind = "str" )
General token kinds
Variables ¶
View Source
var ( MotorolaBin = Prefix("%", Bin) MotorolaOct = Prefix("@", Oct) MotorolaHex = Prefix("$", Hex) GoBin = Rules( Prefix("0b", Bin.WithDigitSep(Rune('_'))), Prefix("0B", Bin.WithDigitSep(Rune('_'))), ) GoOct = Rules( Prefix("0o", Oct.WithDigitSep(Rune('_'))), Prefix("0O", Oct.WithDigitSep(Rune('_'))), ) GoHex = Rules( Prefix("0x", goHex), Prefix("0X", goHex), ) )
View Source
var ( StrDoubleQuote = Str('"', '"') StrSingleQuote = Str('\'', '\'') )
Functions ¶
func FormatTokenTable ¶
Types ¶
type Class ¶
Class returns true if the given rune is a member.
var ( // IsAny returns true as long as there is a rune available in the input // stream. IsAny Class = func(r rune) bool { return r != end } // IsCurrency returns true when given a rune that is a currency symbol as // defined by Unicode. IsCurrency Class = func(r rune) bool { return unicode.Is(unicode.Sc, r) } // IsDigit returns true when given a digit as defined by Unicode. IsDigit Class = unicode.IsDigit // IsDigit01 returns true when given a valid binary digit. IsDigit01 Class = Rune('0', '1') // IsDigit07 returns true when given a valid octal digit. IsDigit07 Class = Range('0', '7') // IsDigit09 returns true when given a valid decimal digit. IsDigit09 Class = Range('0', '9') // IsDigit0F returns true when given a valid hexadecimal digit. IsDigit0F Class = Or( IsDigit09, Range('a', 'f'), Range('A', 'F'), ) // IsLetter returns true when given rune is a letter as defined by Unicode. IsLetter Class = unicode.IsLetter // IsLetterAZ returns true when given letters from the Latin alphabet. IsLetterAZ Class = Or( Range('a', 'z'), Range('A', 'Z'), ) // IsLetterUnder returns true when given letters as defined by Unicode // or an underscore. IsLetterUnder = Or( IsLetter, Rune('_'), ) // IsLetterDigitUnder returns true when given letters as digits as defined // by Unicode or an underscore. IsLetterDigitUnder = Or( IsLetterUnder, IsDigit, ) // IsNone always returns false. IsNone Class = func(r rune) bool { return false } // IsPrintable returns true when given a rune that is printable as defined // by Unicode. IsPrintable Class = unicode.IsPrint // IsRune8 returns true when given a rune that can be represented by // an 8-bit number. IsRune8 Class = Range(0, 0xff) // IsRune16 returns true when given a rune that can be represented by // a 16-bit number. IsRune16 Class = Range(0, 0xffff) IsSign Class = Rune('+', '-') // IsWhitespace returns true when the given rune is whitespace as // defined by Unicode. IsWhitespace Class = unicode.IsSpace )
func Not ¶
Not returns a Class function that returns true when given a rune that does not match class c.
Example ¶
isA := Rune('a') isNotA := Not(isA) fmt.Printf("%c: %v\n", 'a', isNotA('a')) fmt.Printf("%c: %v\n", 'b', isNotA('b'))
Output: a: false b: true
func Or ¶
Or returns a Class that returns true when given a rune that matches any class found in cs.
Example ¶
isLowerAZ := Range('a', 'z') isUpperAZ := Range('A', 'Z') isLetterAZ := Or(isLowerAZ, isUpperAZ) fmt.Printf("%c: %v\n", 'f', isLetterAZ('f')) fmt.Printf("%c: %v\n", 'F', isLetterAZ('F')) fmt.Printf("%c: %v\n", '4', isLetterAZ('4'))
Output: f: true F: true 4: false
func Range ¶
Range returns a Class function that returns true when given a rune that is between from and to inclusive.
Example ¶
isDigit09 := Range('0', '9') fmt.Printf("%c: %v\n", '3', isDigit09('3')) fmt.Printf("%c: %v\n", '6', isDigit09('6')) fmt.Printf("%c: %v\n", 'a', isDigit09('a'))
Output: 3: true 6: true a: false
type CommentRule ¶
type CommentRule struct {
// contains filtered or unexported fields
}
var ( CppLineComment CommentRule = Comment("//", "\n") CppBlockComment CommentRule = Comment("/*", "*/") )
func Comment ¶
func Comment(begin string, end string) CommentRule
func (CommentRule) Eval ¶
func (r CommentRule) Eval(s *Scanner) bool
func (CommentRule) WithKeep ¶
func (r CommentRule) WithKeep(keep *bool) CommentRule
type ExpRule ¶
type ExpRule struct {
// contains filtered or unexported fields
}
var ( Exp ExpRule = NewExpRule(Rune('E', 'e'), IsSign, IsDigit09) HexExp ExpRule = NewExpRule(Rune('P', 'p'), IsSign, IsDigit0F) )
func (ExpRule) WithDigitSep ¶
type FracRule ¶
type FracRule struct {
// contains filtered or unexported fields
}
var ( Frac FracRule = NewFracRule(IsDigit09, Rune('.')) HexFrac FracRule = NewFracRule(IsDigit0F, Rune('.')) )
func NewFracRule ¶
func (FracRule) WithDigitSep ¶
type IdentRule ¶
type IdentRule struct {
// contains filtered or unexported fields
}
var GoIdent IdentRule = Ident(IsLetterUnder, IsLetterDigitUnder)
func (IdentRule) WithKeywords ¶
type IntRule ¶
type IntRule struct {
// contains filtered or unexported fields
}
var ( Bin IntRule = NewIntRule(IsDigit01).WithKind(BinKind) Hex IntRule = NewIntRule(IsDigit0F).WithKind(HexKind) Int IntRule = UInt.WithSign(IsSign) Oct IntRule = NewIntRule(IsDigit07).WithKind(OctKind) UInt IntRule = NewIntRule(IsDigit09) )
func NewIntRule ¶
func (IntRule) WithDigitSep ¶
func (IntRule) WithLeadingDigitSep ¶
func (IntRule) WithLeadingZero ¶
type NumRule ¶
type NumRule struct {
// contains filtered or unexported fields
}
var ( Real NumRule = NewNumRule(Int, Frac) RealExp NumRule = NewNumRule(Int, Frac).WithExpRule(Exp) HexURealExp NumRule = NewNumRule(UInt, HexFrac).WithExpRule(HexExp) HexRealExp NumRule = NewNumRule(Hex, HexFrac).WithExpRule(HexExp) UReal NumRule = NewNumRule(UInt, Frac) URealExp NumRule = UReal.WithExpRule(Exp) )
func NewNumRule ¶
func (NumRule) WithExpRule ¶
type Pos ¶
Pos represents a position within an input stream.
type PrefixRule ¶
type PrefixRule struct {
// contains filtered or unexported fields
}
func Prefix ¶
func Prefix(prefix string, rule Rule) PrefixRule
func (PrefixRule) Eval ¶
func (r PrefixRule) Eval(s *Scanner) bool
type RuleSet ¶
type RuleSet struct {
// contains filtered or unexported fields
}
func (RuleSet) WithDiscards ¶
func (RuleSet) WithPostTokenFunc ¶
func (RuleSet) WithPreTokenFunc ¶
type Scanner ¶
type Scanner struct { Name string This rune Prev rune Kind KindSet // contains filtered or unexported fields }
func NewFromBytes ¶
func NewFromString ¶
func (*Scanner) Emit ¶
Emit returns the token that has been built and resets the builder for the next token.
func (*Scanner) InitFromBytes ¶
InitBytes resets the input stream to src.
func (*Scanner) InitFromString ¶
InitString resets the input stream to src.
func (*Scanner) Keep ¶
func (s *Scanner) Keep()
Keep adds the current rune to the token being built and advances the stream to the next rune.
func (*Scanner) KeepIfPrefix ¶
type StrRule ¶
type StrRule struct {
// contains filtered or unexported fields
}
func (StrRule) WithMultiline ¶
type SuffixRule ¶
type SuffixRule struct {
// contains filtered or unexported fields
}
func Suffix ¶
func Suffix(suffix Class, rule Rule) SuffixRule
func (SuffixRule) Eval ¶
func (r SuffixRule) Eval(s *Scanner) bool
func (SuffixRule) WithKind ¶
func (r SuffixRule) WithKind(k string) SuffixRule
type SymbolRule ¶
type SymbolRule struct {
// contains filtered or unexported fields
}
func Symbol ¶
func Symbol(symbol string) SymbolRule
func (SymbolRule) Eval ¶
func (r SymbolRule) Eval(s *Scanner) bool
func (SymbolRule) WithKind ¶
func (r SymbolRule) WithKind(k string) SymbolRule
Source Files ¶
Click to show internal directories.
Click to hide internal directories.