lexstrings

Documentation

Overview

String lexer package

Derived from template/parse/lex.go Copyright 2011 The Go Authors.

Index

• Variables
• func IsAlphaNumeric(char rune) bool
• func IsSpace(char rune) bool
• type Item
  • func (self Item) String() string
• type ItemType
  • func (self ItemType) String() string
• type Lexer
  • func Lex(input *bufio.Reader, initState StateFn) *Lexer
  • func (self *Lexer) Accept(valid string) (ok bool, err error)
  • func (self *Lexer) AcceptNot(invalid string) (ok bool, err error)
  • func (self *Lexer) AcceptRun(valid string) (err error)
  • func (self *Lexer) AcceptRunNot(invalid string) (err error)
  • func (self *Lexer) Backup() (err error)
  • func (self *Lexer) Emit(t ItemType) Item
  • func (self *Lexer) Errorf(format string, args ...interface{}) (StateFn, Item)
  • func (self *Lexer) Ignore()
  • func (self *Lexer) LineNumber() int
  • func (self *Lexer) LinePosition() int
  • func (self *Lexer) Next() (char rune, err error)
  • func (self *Lexer) NextItem() Item
  • func (self *Lexer) Peek() (char rune, err error)
  • func (self *Lexer) Rewind(state StateFn)
  • func (self *Lexer) ScanNumber() (ok bool, err error)
• type StateFn

Variables

var ItemName = map[ItemType]string{
	Incomplete: "incomplete",
	ItemError:  "error",
	ItemChar:   "char",
	ItemEOF:    "EOF",
	ItemEnd:    "end",
	ItemField:  "field",
	ItemNumber: "number",
	ItemText:   "text",
	ItemID:     "id",
	ItemSeq:    "seq",
	ItemQual:   "qual",
}

Make the types prettyprint.

Functions

func IsAlphaNumeric

func IsAlphaNumeric(char rune) bool

IsAlphaNumeric reports whether char is an alphabetic, digit, or underscore.

func IsSpace

func IsSpace(char rune) bool

IsSpace reports whether char is a space character.

Types

type Item

type Item struct {
	Type ItemType
	Val  string
}

Item represents a token or text string returned from the scanner.

func (Item) String

func (self Item) String() string

type ItemType

type ItemType int

ItemType identifies the type of Lex Items.

const (
	Incomplete ItemType = iota // incomplete item
	ItemError                  // error occurred; value is text of error
	ItemChar                   // printable ASCII character; grab bag for comma etc.
	ItemEOF
	ItemEnd
	ItemField
	ItemNumber
	ItemText
	ItemID
	ItemSeq
	ItemQual
	LastBuiltin
)

func (ItemType) String

func (self ItemType) String() string

type Lexer

type Lexer struct {
	Buffer []rune // buffer to store tokens being built.
	// contains filtered or unexported fields
}

Lexer holds the state of the scanner.

func Lex

func Lex(input *bufio.Reader, initState StateFn) *Lexer

Lex creates a new scanner for the input string.

func (*Lexer) Accept

func (self *Lexer) Accept(valid string) (ok bool, err error)

Accept consumes the next rune if it's from the valid set.

func (*Lexer) AcceptNot

func (self *Lexer) AcceptNot(invalid string) (ok bool, err error)

AcceptNot consumes the next char if it's not from the invalid set.

func (*Lexer) AcceptRun

func (self *Lexer) AcceptRun(valid string) (err error)

AcceptRun consumes a run of runes from the valid set.

func (*Lexer) AcceptRunNot

func (self *Lexer) AcceptRunNot(invalid string) (err error)

AcceptRunNot consumes a run of bytes not from the invalid set.

func (*Lexer) Backup

func (self *Lexer) Backup() (err error)

Backup steps back one rune. Can only be called once per call of next.

func (*Lexer) Emit

func (self *Lexer) Emit(t ItemType) Item

Emit passes an Item back to the client.

func (*Lexer) Errorf

func (self *Lexer) Errorf(format string, args ...interface{}) (StateFn, Item)

Error returns an error token and terminates the scan by passing back a nil pointer that will be the next state, terminating self.run.

func (*Lexer) Ignore

func (self *Lexer) Ignore()

Ignore skips over the pending input before this point.

func (*Lexer) LineNumber

func (self *Lexer) LineNumber() int

LineNumber reports which line we're on.

func (*Lexer) LinePosition

func (self *Lexer) LinePosition() int

LinePosition reports where we are in the line we're on.

func (*Lexer) Next

func (self *Lexer) Next() (char rune, err error)

Next returns the next rune in the input.

func (*Lexer) NextItem

func (self *Lexer) NextItem() Item

NextItem returns the next Item from the input.

func (*Lexer) Peek

func (self *Lexer) Peek() (char rune, err error)

Peek returns but does not consume the next rune in the input.

func (*Lexer) Rewind

func (self *Lexer) Rewind(state StateFn)

Rewind changes the Lexer to another state - presumably the initial state.

func (*Lexer) ScanNumber

func (self *Lexer) ScanNumber() (ok bool, err error)

ScanNumber scans a number: decimal, octal, hex, or float. This isn't a perfect number scanner - for instance it accepts "." and "0x0.2" and "089" - but when it's wrong the input is invalid and the parser (via strconv) will notice.

func LexNumber(self *Lexer) {
	if !self.ScanNumber() {
		return self.Errorf("bad number syntax: %q", self.input[self.start:self.pos])
	}
	self.Emit(ItemNumber)
	return <nextStateFn>
}

type StateFn

type StateFn func(*Lexer) (StateFn, Item)

StateFn represents the state of the scanner as a function that returns the next state.