lexer

Documentation

Index

• func BlockStringValue(s []byte) (formatted []byte)
• type Input
  • func NewInput(bs []byte) *Input
  • func (i *Input) PeekOne(n int) rune
  • func (i *Input) PeekOneRune(n int) (rune, int)
  • func (i *Input) Reset()
  • func (i *Input) Value(t *Token) []byte
• type Lexer
  • func NewLexer(in *Input) *Lexer
  • func (l *Lexer) Read() (t Token)
• type Token
• type TokenKind
  • func (tk TokenKind) String() string

Functions

func BlockStringValue

func BlockStringValue(s []byte) (formatted []byte)

Types

type Input

type Input struct {
	Pos    int
	Line   int
	Column int
	// contains filtered or unexported fields
}

func NewInput

func NewInput(bs []byte) *Input

func (*Input) PeekOne

func (i *Input) PeekOne(n int) rune

func (*Input) PeekOneRune

func (i *Input) PeekOneRune(n int) (rune, int)

func (*Input) Reset

func (i *Input) Reset()

func (*Input) Value

func (i *Input) Value(t *Token) []byte

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(in *Input) *Lexer

func (*Lexer) Read

func (l *Lexer) Read() (t Token)

type Token

type Token struct {
	Kind TokenKind

	Value string

	Err        error
	Start, End int
	Line, Col  int
}

type TokenKind

type TokenKind int

TokenKind tells what kind of value is in the Token

const (
	// Undefined token
	Undefined TokenKind = iota
	// EOFToken -> EOF
	EOFToken
	// PunctuatorToken has special characters
	PunctuatorToken
	// NameToken has names
	NameToken // /[_A-Za-z][_0-9A-Za-z]*/
	// IntValueToken has iteger numbers
	IntValueToken // NegativeSign(opt) | NonZeroDigit | Digit (list, opt)
	// FloatValueToken has float numbers
	FloatValueToken // Sign (opt) | IntegerPart | FractionalPart (ExponentPart)
	// StringValueToken has string values
	StringValueToken // "something which is a string" | """this is also valid"""
)

func (TokenKind) String

func (tk TokenKind) String() string