token

Documentation

Index

• type Location
  • func (loc *Location) String() string
• type LocationError
  • func (err *LocationError) Error() string
• type Rune
  • func (r Rune) IsRuneError() bool
• type Scanner
  • func NewScanner(file string, r io.Reader) *Scanner
  • func (s *Scanner) Accept(fn func(rune) bool) bool
  • func (s *Scanner) AcceptAny(charset string) bool
  • func (s *Scanner) AcceptDigit() bool
  • func (s *Scanner) AcceptRune(c rune) bool
  • func (s *Scanner) AcceptSeq(fn func(rune) bool) int
  • func (s *Scanner) AcceptSeqAny(charset string) int
  • func (s *Scanner) AcceptSeqDigit() int
  • func (s *Scanner) AcceptSeqRune(c rune) int
  • func (s *Scanner) AcceptSeqSpace() int
  • func (s *Scanner) AcceptSpace() bool
  • func (s *Scanner) AcceptString(literal string) (int, bool)
  • func (s *Scanner) EOF() bool
  • func (s *Scanner) EmitToken(typ Type) *Token
  • func (s *Scanner) Err() error
  • func (s *Scanner) Ignore()
  • func (s *Scanner) Loc() *Location
  • func (s *Scanner) LocStart() *Location
  • func (s *Scanner) Peek() (rune, bool)
  • func (s *Scanner) Rune() rune
  • func (s *Scanner) ScanRune() error
  • func (s *Scanner) SetPath(path string)
  • func (s *Scanner) Text() string
• type Source
• type Token
• type Type
  • func (typ Type) String() string

Types

type Location

type Location struct {
	File string // a name representing the source stream
	Path string // a physical location which may differ from File
	Pos  int
	Line int // line number (starting at 1 when tracked)
	Col  int // line column number (starting at 1 when tracked)
}

func (*Location) String

func (loc *Location) String() string

type LocationError

type LocationError struct {
	Err    error
	Source *Location
}

func (*LocationError) Error

func (err *LocationError) Error() string

type Rune

type Rune struct {
	C rune
	N int
}

Rune contains a rune that read by Scanner during peeking operations.

func (Rune) IsRuneError

func (r Rune) IsRuneError() bool

IsRuneError returns true if Rune represents an invalid utf-8 sequence read by utf8.DecodeRune.

type Scanner

type Scanner struct {
	// contains filtered or unexported fields
}

Scanner facilitates construction of tokens from a byte stream (io.Reader).

func NewScanner

func NewScanner(file string, r io.Reader) *Scanner

NewScanner initializes and returns a new Scanner.

func (*Scanner) Accept

func (s *Scanner) Accept(fn func(rune) bool) bool

func (*Scanner) AcceptAny

func (s *Scanner) AcceptAny(charset string) bool

func (*Scanner) AcceptDigit

func (s *Scanner) AcceptDigit() bool

func (*Scanner) AcceptRune

func (s *Scanner) AcceptRune(c rune) bool

func (*Scanner) AcceptSeq

func (s *Scanner) AcceptSeq(fn func(rune) bool) int

func (*Scanner) AcceptSeqAny

func (s *Scanner) AcceptSeqAny(charset string) int

func (*Scanner) AcceptSeqDigit

func (s *Scanner) AcceptSeqDigit() int

func (*Scanner) AcceptSeqRune

func (s *Scanner) AcceptSeqRune(c rune) int

func (*Scanner) AcceptSeqSpace

func (s *Scanner) AcceptSeqSpace() int

func (*Scanner) AcceptSpace

func (s *Scanner) AcceptSpace() bool

func (*Scanner) AcceptString

func (s *Scanner) AcceptString(literal string) (int, bool)

func (*Scanner) EOF

func (s *Scanner) EOF() bool

func (*Scanner) EmitToken

func (s *Scanner) EmitToken(typ Type) *Token

EmitToken returns a token containing the text scanned since the last call to either EmitToken or Ignore.

func (*Scanner) Err

func (s *Scanner) Err() error

Err returns an error encountered during the last read on the input stream. Err will always return false while there are still buffered runes that need to be accepted.

func (*Scanner) Ignore

func (s *Scanner) Ignore()

Ignore causes the scanner to skip all text scanned since the last call to either EmitToken or Ignore.

func (*Scanner) Loc

func (s *Scanner) Loc() *Location

Loc returns a Location referencing the current scanner position, the last position of the current token.

func (*Scanner) LocStart

func (s *Scanner) LocStart() *Location

LocStart returns a Location referencing the beginning of the current token, just beyond the end of the previous token.

func (*Scanner) Peek

func (s *Scanner) Peek() (rune, bool)

Peek returns the next rune to be scanned, if there are any. If an invalid utf-8 sequence or EOF prevents futher runes from being scanned Peek returns a false second value. If Peek returns a false value the next call to s.ScanRune will return an error that reflects of the cause.

func (*Scanner) Rune

func (s *Scanner) Rune() rune

Rune returns the current unicode rune that is being scanned. The rune returned by Rune is the last rune in a token returned by EmitToken.

func (*Scanner) ScanRune

func (s *Scanner) ScanRune() error

ScanRune attempts to scan a utf-8 rune from the input for inclusion in the current token. If an error prevents a valid unicode rune from being scanned then an error will be returned.

func (*Scanner) SetPath

func (s *Scanner) SetPath(path string)

SetPath associates a physical location (e.g. filesystem path) with s to aid in debugging projects which scan many ungrouped files.

func (*Scanner) Text

func (s *Scanner) Text() string

Text returns a string containing text scanned since the last call to either EmitToken or Ignore.

type Source

type Source interface {
	// Token returns the current token.  Token returns nil if Scan has not been
	// called.
	Token() *Token
	// Peek returns the next token in the stream.  At the end of the stream
	// Peek should return a value to indicate the lack of a token (EOF).
	Peek() *Token
	// Scan advances the token stream if possible.  If there are no tokens
	// remaining Scan returns false.
	Scan() bool
}

Source is an abstract stream of tokens which allows one token lookahead.

type Token

type Token struct {
	Type   Type
	Text   string
	Source *Location
}

type Type

type Type uint

const (
	INVALID Type = iota
	ERROR
	EOF

	HASH_BANG

	// Atomic expressions & literals
	SYMBOL
	INT
	INT_OCTAL_MACRO
	INT_OCTAL
	INT_HEX_MACRO
	INT_HEX
	FLOAT
	STRING
	STRING_RAW

	COMMENT

	// Operators
	NEGATIVE // arithmetic negation is parsed specially
	QUOTE
	UNBOUND
	FUN_REF

	// Delimiters
	PAREN_L
	PAREN_R
	BRACE_L
	BRACE_R
)

Type constants used for the elps lexer/parser. These constants aren't necessary to use the package.

func (Type) String

func (typ Type) String() string