scanner

package module
v0.0.0-...-f8b1171 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 17, 2023 License: MIT Imports: 13 Imported by: 0

README

scanner

Work in Progress

Documentation

Index

Examples

Constants

View Source
const (
	BinKind     = "bin"
	CommentKind = "comment"
	ErrKind     = "err"
	RealKind    = "real"
	ExpKind     = "exp"
	HexKind     = "hex"
	IdentKind   = "ident"
	IllegalKind = "illegal"
	IntKind     = "int"
	NumKind     = "num"
	OctKind     = "oct"
	StrKind     = "str"
)

General token kinds

Variables

View Source
var (
	MotorolaBin = Prefix("%", Bin)
	MotorolaOct = Prefix("@", Oct)
	MotorolaHex = Prefix("$", Hex)

	GoBin = Rules(
		Prefix("0b", Bin.WithDigitSep(Rune('_'))),
		Prefix("0B", Bin.WithDigitSep(Rune('_'))),
	)
	GoOct = Rules(
		Prefix("0o", Oct.WithDigitSep(Rune('_'))),
		Prefix("0O", Oct.WithDigitSep(Rune('_'))),
	)
	GoHex = Rules(
		Prefix("0x", goHex),
		Prefix("0X", goHex),
	)
)
View Source
var (
	StrDoubleQuote = Str('"', '"')
	StrSingleQuote = Str('\'', '\'')
)

Functions

func FormatTokenTable

func FormatTokenTable(ts []Token) string

func Main

func Main(fn func(io.Reader) []Token)

func MainWithArgs

func MainWithArgs(fn func(io.Reader) []Token, args []string)

func Repeat

func Repeat(fn func(), n int)

Repeat calls function fn in a loop for n times.

func RunTests

func RunTests(t *testing.T, rules RuleSet, tests []TestCase)

func While

func While(s *Scanner, c Class, fn func())

Types

type Class

type Class func(rune) bool

Class returns true if the given rune is a member.

var (
	// IsAny returns true as long as there is a rune available in the input
	// stream.
	IsAny Class = func(r rune) bool {
		return r != end
	}

	// IsCurrency returns true when given a rune that is a currency symbol as
	// defined by Unicode.
	IsCurrency Class = func(r rune) bool {
		return unicode.Is(unicode.Sc, r)
	}

	// IsDigit returns true when given a digit as defined by Unicode.
	IsDigit Class = unicode.IsDigit

	// IsDigit01 returns true when given a valid binary digit.
	IsDigit01 Class = Rune('0', '1')

	// IsDigit07 returns true when given a valid octal digit.
	IsDigit07 Class = Range('0', '7')

	// IsDigit09 returns true when given a valid decimal digit.
	IsDigit09 Class = Range('0', '9')

	// IsDigit0F returns true when given a valid hexadecimal digit.
	IsDigit0F Class = Or(
		IsDigit09,
		Range('a', 'f'),
		Range('A', 'F'),
	)

	// IsLetter returns true when given rune is a letter as defined by Unicode.
	IsLetter Class = unicode.IsLetter

	// IsLetterAZ returns true when given letters from the Latin alphabet.
	IsLetterAZ Class = Or(
		Range('a', 'z'),
		Range('A', 'Z'),
	)

	// IsLetterUnder returns true when given letters as defined by Unicode
	// or an underscore.
	IsLetterUnder = Or(
		IsLetter,
		Rune('_'),
	)

	// IsLetterDigitUnder returns true when given letters as digits as defined
	// by Unicode or an underscore.
	IsLetterDigitUnder = Or(
		IsLetterUnder,
		IsDigit,
	)

	// IsNone always returns false.
	IsNone Class = func(r rune) bool {
		return false
	}

	// IsPrintable returns true when given a rune that is printable as defined
	// by Unicode.
	IsPrintable Class = unicode.IsPrint

	// IsRune8 returns true when given a rune that can be represented by
	// an 8-bit number.
	IsRune8 Class = Range(0, 0xff)

	// IsRune16 returns true when given a rune that can be represented by
	// a 16-bit number.
	IsRune16 Class = Range(0, 0xffff)

	IsSign Class = Rune('+', '-')

	// IsWhitespace returns true when the given rune is whitespace as
	// defined by Unicode.
	IsWhitespace Class = unicode.IsSpace
)

func Not

func Not(c Class) Class

Not returns a Class function that returns true when given a rune that does not match class c.

Example
isA := Rune('a')
isNotA := Not(isA)
fmt.Printf("%c: %v\n", 'a', isNotA('a'))
fmt.Printf("%c: %v\n", 'b', isNotA('b'))
Output:

a: false
b: true

func Or

func Or(cs ...Class) Class

Or returns a Class that returns true when given a rune that matches any class found in cs.

Example
isLowerAZ := Range('a', 'z')
isUpperAZ := Range('A', 'Z')
isLetterAZ := Or(isLowerAZ, isUpperAZ)
fmt.Printf("%c: %v\n", 'f', isLetterAZ('f'))
fmt.Printf("%c: %v\n", 'F', isLetterAZ('F'))
fmt.Printf("%c: %v\n", '4', isLetterAZ('4'))
Output:

f: true
F: true
4: false

func Range

func Range(from rune, to rune) Class

Range returns a Class function that returns true when given a rune that is between from and to inclusive.

Example
isDigit09 := Range('0', '9')
fmt.Printf("%c: %v\n", '3', isDigit09('3'))
fmt.Printf("%c: %v\n", '6', isDigit09('6'))
fmt.Printf("%c: %v\n", 'a', isDigit09('a'))
Output:

3: true
6: true
a: false

func Rune

func Rune(rs ...rune) Class

Rune returns a Class function that returns true when given any rune found in rs.

Example
isAB := Rune('a', 'b')
fmt.Printf("%c: %v\n", 'a', isAB('a'))
fmt.Printf("%c: %v\n", 'b', isAB('b'))
fmt.Printf("%c: %v\n", 'c', isAB('c'))
Output:

a: true
b: true
c: false

type CommentRule

type CommentRule struct {
	// contains filtered or unexported fields
}
var (
	CppLineComment  CommentRule = Comment("//", "\n")
	CppBlockComment CommentRule = Comment("/*", "*/")
)

func Comment

func Comment(begin string, end string) CommentRule

func (CommentRule) Eval

func (r CommentRule) Eval(s *Scanner) bool

func (CommentRule) WithKeep

func (r CommentRule) WithKeep(keep *bool) CommentRule

type ExpRule

type ExpRule struct {
	// contains filtered or unexported fields
}
var (
	Exp    ExpRule = NewExpRule(Rune('E', 'e'), IsSign, IsDigit09)
	HexExp ExpRule = NewExpRule(Rune('P', 'p'), IsSign, IsDigit0F)
)

func NewExpRule

func NewExpRule(isExp Class, isSign Class, isDigit Class) ExpRule

func (ExpRule) Eval

func (r ExpRule) Eval(scan *Scanner) bool

func (ExpRule) WithDigitSep

func (r ExpRule) WithDigitSep(c Class) ExpRule

func (ExpRule) WithEmpty

func (r ExpRule) WithEmpty(b bool) ExpRule

type FracRule

type FracRule struct {
	// contains filtered or unexported fields
}
var (
	Frac    FracRule = NewFracRule(IsDigit09, Rune('.'))
	HexFrac FracRule = NewFracRule(IsDigit0F, Rune('.'))
)

func NewFracRule

func NewFracRule(isDigit Class, isDecSep Class) FracRule

func (FracRule) Eval

func (r FracRule) Eval(scan *Scanner) bool

func (FracRule) WithDigitSep

func (r FracRule) WithDigitSep(c Class) FracRule

func (FracRule) WithEmpty

func (r FracRule) WithEmpty(b bool) FracRule

type IdentRule

type IdentRule struct {
	// contains filtered or unexported fields
}

func Ident

func Ident(head Class, tail Class) IdentRule

func (IdentRule) Eval

func (r IdentRule) Eval(s *Scanner) bool

func (IdentRule) WithKeywords

func (r IdentRule) WithKeywords(ks ...string) IdentRule

type IntRule

type IntRule struct {
	// contains filtered or unexported fields
}
var (
	Bin  IntRule = NewIntRule(IsDigit01).WithKind(BinKind)
	Hex  IntRule = NewIntRule(IsDigit0F).WithKind(HexKind)
	Int  IntRule = UInt.WithSign(IsSign)
	Oct  IntRule = NewIntRule(IsDigit07).WithKind(OctKind)
	UInt IntRule = NewIntRule(IsDigit09)
)

func NewIntRule

func NewIntRule(isDigit Class) IntRule

func (IntRule) Eval

func (r IntRule) Eval(scan *Scanner) bool

func (IntRule) WithDigitSep

func (r IntRule) WithDigitSep(c Class) IntRule

func (IntRule) WithKind

func (r IntRule) WithKind(k string) IntRule

func (IntRule) WithLeadingDigitSep

func (r IntRule) WithLeadingDigitSep(b bool) IntRule

func (IntRule) WithLeadingZero

func (r IntRule) WithLeadingZero(b bool) IntRule

func (IntRule) WithSign

func (r IntRule) WithSign(c Class) IntRule

type KindSet

type KindSet []string

func NewKindSet

func NewKindSet(kinds ...string) KindSet

func (*KindSet) Add

func (k *KindSet) Add(kinds ...string)

func (KindSet) Contains

func (k KindSet) Contains(k2 KindSet) bool

func (KindSet) ContainsAny

func (k KindSet) ContainsAny(kinds ...string) bool

func (*KindSet) Remove

func (k *KindSet) Remove(kinds ...string)

func (KindSet) String

func (k KindSet) String() string

type NumRule

type NumRule struct {
	// contains filtered or unexported fields
}
var (
	Real        NumRule = NewNumRule(Int, Frac)
	RealExp     NumRule = NewNumRule(Int, Frac).WithExpRule(Exp)
	HexURealExp NumRule = NewNumRule(UInt, HexFrac).WithExpRule(HexExp)
	HexRealExp  NumRule = NewNumRule(Hex, HexFrac).WithExpRule(HexExp)
	UReal       NumRule = NewNumRule(UInt, Frac)
	URealExp    NumRule = UReal.WithExpRule(Exp)
)

func NewNumRule

func NewNumRule(intRule IntRule, fracRule FracRule) NumRule

func (NumRule) Eval

func (r NumRule) Eval(scan *Scanner) bool

func (NumRule) WithExpRule

func (r NumRule) WithExpRule(er ExpRule) NumRule

type Pos

type Pos struct {
	Name   string `json:",omitempty"`
	Line   int
	Column int
}

Pos represents a position within an input stream.

func NewPos

func NewPos(name string) Pos

NewPos returns a new position with the given name and the line number and column number set to one.

func (Pos) String

func (p Pos) String() string

type PrefixRule

type PrefixRule struct {
	// contains filtered or unexported fields
}

func Prefix

func Prefix(prefix string, rule Rule) PrefixRule

func (PrefixRule) Eval

func (r PrefixRule) Eval(s *Scanner) bool

type Rule

type Rule interface {
	Eval(*Scanner) bool
}

func NewRule

func NewRule(eval func(*Scanner) bool) Rule

type RuleSet

type RuleSet struct {
	// contains filtered or unexported fields
}

func Rules

func Rules(rules ...Rule) RuleSet

func SymbolMap

func SymbolMap(sk map[string]string) RuleSet

func Symbols

func Symbols(ss ...string) RuleSet

func (RuleSet) Eval

func (r RuleSet) Eval(scan *Scanner) bool

func (RuleSet) Next

func (r RuleSet) Next(scan *Scanner) Token

func (RuleSet) WithDiscards

func (r RuleSet) WithDiscards(c Class) RuleSet

func (RuleSet) WithPostTokenFunc

func (r RuleSet) WithPostTokenFunc(fn func(*Scanner, Token) Token) RuleSet

func (RuleSet) WithPreTokenFunc

func (r RuleSet) WithPreTokenFunc(fn func(*Scanner)) RuleSet

type Runner

type Runner struct {
	// contains filtered or unexported fields
}

func NewRunner

func NewRunner(scan *Scanner, rules RuleSet) *Runner

func (*Runner) All

func (r *Runner) All() []Token

func (*Runner) More

func (r *Runner) More() bool

func (*Runner) Next

func (r *Runner) Next() Token

func (*Runner) Peek

func (r *Runner) Peek() Token

type Scanner

type Scanner struct {
	Name string
	This rune
	Prev rune
	Kind KindSet
	// contains filtered or unexported fields
}

func New

func New(src io.Reader) *Scanner

func NewFromBytes

func NewFromBytes(src []byte) *Scanner

func NewFromString

func NewFromString(src string) *Scanner

func (*Scanner) Discard

func (s *Scanner) Discard()

func (*Scanner) Emit

func (s *Scanner) Emit() Token

Emit returns the token that has been built and resets the builder for the next token.

func (*Scanner) Illegal

func (s *Scanner) Illegal(format string, args ...any)

func (*Scanner) Init

func (s *Scanner) Init(src io.Reader)

Init resets the input stream to src.

func (*Scanner) InitFromBytes

func (s *Scanner) InitFromBytes(src []byte)

InitBytes resets the input stream to src.

func (*Scanner) InitFromString

func (s *Scanner) InitFromString(src string)

InitString resets the input stream to src.

func (*Scanner) Keep

func (s *Scanner) Keep()

Keep adds the current rune to the token being built and advances the stream to the next rune.

func (*Scanner) KeepIfPrefix

func (s *Scanner) KeepIfPrefix(prefix string) bool

func (*Scanner) Lit

func (s *Scanner) Lit() string

func (*Scanner) More

func (s *Scanner) More() bool

func (*Scanner) Peek

func (s *Scanner) Peek(n int) rune

func (*Scanner) PeekTo

func (s *Scanner) PeekTo(n int) string

func (*Scanner) Skip

func (s *Scanner) Skip()

Skip discards the current rune and advances the stream to the next rune.

func (*Scanner) Val

func (s *Scanner) Val() string

func (*Scanner) WriteRune

func (s *Scanner) WriteRune(r rune)

type StrRule

type StrRule struct {
	// contains filtered or unexported fields
}

func Str

func Str(begin rune, end rune) StrRule

func (StrRule) Eval

func (r StrRule) Eval(s *Scanner) bool

func (StrRule) WithKind

func (r StrRule) WithKind(k string) StrRule

func (StrRule) WithMultiline

func (r StrRule) WithMultiline() StrRule

type SuffixRule

type SuffixRule struct {
	// contains filtered or unexported fields
}

func Suffix

func Suffix(suffix Class, rule Rule) SuffixRule

func (SuffixRule) Eval

func (r SuffixRule) Eval(s *Scanner) bool

func (SuffixRule) WithKind

func (r SuffixRule) WithKind(k string) SuffixRule

type SymbolRule

type SymbolRule struct {
	// contains filtered or unexported fields
}

func Symbol

func Symbol(symbol string) SymbolRule

func (SymbolRule) Eval

func (r SymbolRule) Eval(s *Scanner) bool

func (SymbolRule) WithKind

func (r SymbolRule) WithKind(k string) SymbolRule

type TestCase

type TestCase struct {
	// contains filtered or unexported fields
}

func NewTestCase

func NewTestCase(src string, val string, kinds ...string) TestCase

func (TestCase) And

func (t TestCase) And(val string, kinds ...string) TestCase

type Token

type Token struct {
	Val  string
	Lit  string `json:",omitempty"`
	Kind KindSet
	Pos  Pos
	Err  TokenError
}

func (Token) IsKind

func (t Token) IsKind(k ...string) bool

func (Token) String

func (t Token) String() string

type TokenError

type TokenError interface {
	Pos() Pos
	Message() string
	Error() string
	Unwrap() error
}

Directories

Path Synopsis
examples
dms
go
internal

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL