tokenizer

Documentation

Index

• func Split(tokens []*Token, delimiter TokenType) [][]*Token
• func Stringify(tokens []*Token) string
• type Token
  • func NewToken(tp, text string) *Token
  • func Tokenize(text string) []*Token
  • func (t *Token) String() string
• type TokenType

Functions

func Split

func Split(tokens []*Token, delimiter TokenType) [][]*Token

func Stringify

func Stringify(tokens []*Token) string

Types

type Token

type Token struct {
	Type  TokenType
	Value string
}

func NewToken

func NewToken(tp, text string) *Token

func Tokenize

func Tokenize(text string) []*Token

func (*Token) String

func (t *Token) String() string

type TokenType

type TokenType = string

const (
	Underscore         TokenType = "_"
	Asterisk           TokenType = "*"
	PoundSign          TokenType = "#"
	Backtick           TokenType = "`"
	LeftSquareBracket  TokenType = "["
	RightSquareBracket TokenType = "]"
	LeftParenthesis    TokenType = "("
	RightParenthesis   TokenType = ")"
	ExclamationMark    TokenType = "!"
	Tilde              TokenType = "~"
	Hyphen             TokenType = "-"
	PlusSign           TokenType = "+"
	Dot                TokenType = "."
	LessThan           TokenType = "<"
	GreaterThan        TokenType = ">"
	DollarSign         TokenType = "$"
	EqualSign          TokenType = "="
	Pipe               TokenType = "|"
	Colon              TokenType = ":"
	Backslash          TokenType = "\\"
	Newline            TokenType = "\n"
	Space              TokenType = " "
)

const (
	Number TokenType = "number"
	Text   TokenType = ""
)