lexer

README

lexer

Generic tokenizer written in Go

Documentation

Index

• func IsRegexOp(t Token) bool
• func LoadTokenMap(keywordTokens map[Token]string)
• func ScanBareIdent(r io.RuneScanner) string
• func ScanDelimited(r io.RuneScanner, start, end rune, escapes map[rune]rune, escapesPassThru bool) ([]byte, error)
• func ScanString(r io.RuneScanner) (string, error)
• type Pos
• type Scanner
  • func NewScanner(r io.Reader) *Scanner
  • func (s *Scanner) Peek() rune
  • func (s *Scanner) Scan() (tok Token, pos Pos, lit string)
  • func (s *Scanner) ScanRegex() (tok Token, pos Pos, lit string)
• type Token
  • func Lookup(ident string) Token
  • func (tok Token) Precedence() int
  • func (tok Token) String() string

Functions

func IsRegexOp

func IsRegexOp(t Token) bool

IsRegexOp returns true if the operator accepts a regex operand.

func LoadTokenMap

func LoadTokenMap(keywordTokens map[Token]string)

LoadTokenMap allows for extra keywords to be added to the lexer

func ScanBareIdent

func ScanBareIdent(r io.RuneScanner) string

ScanBareIdent reads bare identifier from a rune reader.

func ScanDelimited

func ScanDelimited(r io.RuneScanner, start, end rune, escapes map[rune]rune, escapesPassThru bool) ([]byte, error)

func ScanString

func ScanString(r io.RuneScanner) (string, error)

ScanString reads a quoted string from a rune reader.

Types

type Pos

type Pos struct {
	Line int
	Char int
}

Pos specifies the line and character position of a token. The Char and Line are both zero-based indexes.

type Scanner

type Scanner struct {
	// contains filtered or unexported fields
}

Scanner represents a lexical scanner for InfluxQL.

func NewScanner

func NewScanner(r io.Reader) *Scanner

NewScanner returns a new instance of Scanner.

func (*Scanner) Peek

func (s *Scanner) Peek() rune

Peek returns the next rune without advancing the scanner

func (*Scanner) Scan

func (s *Scanner) Scan() (tok Token, pos Pos, lit string)

Scan returns the next token and position from the underlying reader. Also returns the literal text read for strings and numbers tokens since these token types can have different literal representations.

func (*Scanner) ScanRegex

func (s *Scanner) ScanRegex() (tok Token, pos Pos, lit string)

ScanRegex tokenizes a regex expression

type Token

type Token int

Token represents each lexer symbol

const (
	ILLEGAL Token = iota
	EOF
	WS

	LPAREN      // (
	RPAREN      // )
	LBRACKET    // [
	RBRACKET    // ]
	LCURLY      // {
	RCURLY      // }
	COMMA       // ,
	SEMICOLON   // ;
	COLON       // :
	DOT         // .
	SINGLEQUOTE // '
	DOUBLEQUOTE // "
	PERCENT     // %
	DOLLAR      // $
	HASH        // #
	ATSIGN      // @

	IDENT
	NUMBER
	STRING
	BADSTRING
	BADESCAPE
	TRUE
	FALSE
	REGEX
	BADREGEX
	DURATION_VAL

	PLUS      // +
	MINUS     // -
	MUL       // *
	DIV       // /
	AMPERSAND // &
	XOR       // ^
	PIPE      // |
	LSHIFT    // <<
	RSHIFT    // >>

	AND // AND
	OR  // OR

	EQ       // =
	NEQ      // !=
	EQREGEX  // =~
	NEQREGEX // !~
	LT       // <
	LTE      // <=
	GT       // >
	GTE      // >=

)

Token enums

func Lookup

func Lookup(ident string) Token

Lookup returns the token associated with a given string.

func (Token) Precedence

func (tok Token) Precedence() int

Precedence returns the operator precedence of the binary operator token.

func (Token) String

func (tok Token) String() string

String returns the string representation of the token.