lexer

Documentation

Index

• Constants
• func IsKeyword(candidate string) bool
• type Lexeme
• type Lexer
  • func Lex(source input.Source, input string) *Lexer
  • func (l *Lexer) Close()
• type PeekableLexer
  • func NewPeekableLexer(lex *Lexer) *PeekableLexer
  • func (l *PeekableLexer) Close()
  • func (l *PeekableLexer) NextToken() Lexeme
• type TokenType
  • func (i TokenType) String() string

Constants

const EOFRUNE = -1

Functions

func IsKeyword

func IsKeyword(candidate string) bool

IsKeyword returns whether the specified input string is a reserved keyword.

Types

type Lexeme

type Lexeme struct {
	Kind     TokenType          // The type of this lexeme.
	Position input.BytePosition // The starting position of this token in the input string.
	Value    string             // The textual value of this token.
	Error    string             // The error associated with the lexeme, if any.
}

Lexeme represents a token returned from scanning the contents of a file.

type Lexer

type Lexer struct {
	sync.RWMutex
	// contains filtered or unexported fields
}

Lexer holds the state of the scanner.

func Lex

func Lex(source input.Source, input string) *Lexer

Lex creates a new scanner for the input string.

func (*Lexer) Close

func (l *Lexer) Close()

Close stops the lexer from running.

type PeekableLexer

type PeekableLexer struct {
	// contains filtered or unexported fields
}

PeekableLexer wraps a lexer and provides the ability to peek forward without losing state.

func NewPeekableLexer

func NewPeekableLexer(lex *Lexer) *PeekableLexer

NewPeekableLexer returns a new PeekableLexer for the given lexer.

func (*PeekableLexer) Close

func (l *PeekableLexer) Close()

Close stops the lexer from running.

func (*PeekableLexer) NextToken

func (l *PeekableLexer) NextToken() Lexeme

NextToken returns the next token found in the lexer.

type TokenType

type TokenType int

TokenType identifies the type of lexer lexemes.

const (
	TokenTypeError TokenType = iota // error occurred; value is text of error

	// Synthetic semicolon
	TokenTypeSyntheticSemicolon

	TokenTypeEOF
	TokenTypeWhitespace
	TokenTypeSinglelineComment
	TokenTypeMultilineComment
	TokenTypeNewline

	TokenTypeKeyword    // interface
	TokenTypeIdentifier // helloworld
	TokenTypeNumber     // 123

	TokenTypeLeftBrace  // {
	TokenTypeRightBrace // }
	TokenTypeLeftParen  // (
	TokenTypeRightParen // )

	TokenTypePipe  // |
	TokenTypePlus  // +
	TokenTypeMinus // -
	TokenTypeAnd   // &
	TokenTypeDiv   // /

	TokenTypeEquals     // =
	TokenTypeColon      // :
	TokenTypeSemicolon  // ;
	TokenTypeRightArrow // ->
	TokenTypeHash       // #
	TokenTypeEllipsis   // ...
	TokenTypeStar       // *

	// Additional tokens for CEL: https://github.com/google/cel-spec/blob/master/doc/langdef.md#syntax
	TokenTypeQuestionMark       // ?
	TokenTypeConditionalOr      // ||
	TokenTypeConditionalAnd     // &&
	TokenTypeExclamationPoint   // !
	TokenTypeLeftBracket        // [
	TokenTypeRightBracket       // ]
	TokenTypePeriod             // .
	TokenTypeComma              // ,
	TokenTypePercent            // %
	TokenTypeLessThan           // <
	TokenTypeGreaterThan        // >
	TokenTypeLessThanOrEqual    // <=
	TokenTypeGreaterThanOrEqual // >=
	TokenTypeEqualEqual         // ==
	TokenTypeNotEqual           // !=
	TokenTypeString             // "...", '...', """...""", ”'...”'
)

func (TokenType) String

func (i TokenType) String() string