tokenizer

Documentation

Index

• type TextToken
• type Token
• type TokenType
• type Tokenizer
  • func Open(path string) (*Tokenizer, error)
  • func (t *Tokenizer) Next() Token
  • func (t *Tokenizer) Peek() Token

Types

type TextToken

type TextToken struct {
	Token
	Value string
}

type Token

type Token struct {
	Type  TokenType
	Name  string
	Value interface{}
	Pos   scanner.Position
}

type TokenType

type TokenType string

const (
	Identifier  TokenType = "Identifier"
	OpenCurly   TokenType = "OpenCurly"
	CloseCurly  TokenType = "CloseCurly"
	OpenSquare  TokenType = "OpenSquare"
	CloseSquare TokenType = "CloseSquare"
	Dollar      TokenType = "Dollar"
	EOF         TokenType = "EOF"
	NewLine     TokenType = "NewLine"
	Percent     TokenType = "Percent"
	Text        TokenType = "Text"
	Unknown     TokenType = "Unknown"
)

type Tokenizer

type Tokenizer struct {
	OriginalFilePath string
	// contains filtered or unexported fields
}

func Open

func Open(path string) (*Tokenizer, error)

func (*Tokenizer) Next

func (t *Tokenizer) Next() Token

func (*Tokenizer) Peek

func (t *Tokenizer) Peek() Token