lexer

Documentation

Index

• type Lexer
  • func NewLexer(input []byte) Lexer
  • func (l *Lexer) Len() int
  • func (l *Lexer) Next() (int, error)
  • func (l *Lexer) Reset()
  • func (l *Lexer) String() string
  • func (l *Lexer) Token() Token
  • func (l *Lexer) Tokens() []Token
• type Token
• type TokenType
  • func (t TokenType) String() string

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(input []byte) Lexer

func (*Lexer) Len

func (l *Lexer) Len() int

Len returns the number of tokens that have been read.

func (*Lexer) Next

func (l *Lexer) Next() (int, error)

Next reads the next token from the input and returns the number of bytes read.

func (*Lexer) Reset

func (l *Lexer) Reset()

Reset sets points to the beginning of the input and clears the tokens.

func (*Lexer) String

func (l *Lexer) String() string

func (*Lexer) Token

func (l *Lexer) Token() Token

Token returns the last token that was read.

func (*Lexer) Tokens

func (l *Lexer) Tokens() []Token

Tokens returns a copy of all tokens that have been read.

type Token

type Token struct {
	Type    TokenType
	Literal string
	Line    int
	// Col is the UTF-8 character position on the token's line.
	Col int
	// Pos is the byte position in the input where this token starts.
	Pos int
}

type TokenType

type TokenType int

const (
	EOF TokenType = iota
	NEWLINE
	WHITESPACE
	QUOTE
	LBRACE
	RBRACE
	LBRACKET
	RBRACKET
	COMMA
	COLON
	NUMBER
	STRING
	TRUE
	FALSE
	NULL
)

func (TokenType) String

func (t TokenType) String() string