tokenizer

Documentation

Index

• Constants
• type Token
  • func Tokenize(input string) (tokens []Token, err error)

Constants

const (
	TokLabel tokenKind = iota + 1
	TokStringLiteral
	TokLBrace
	TokRBrace
	TokComma
	TokEq
	TokNe
	TokIn
	TokNot
	TokNotIn
	TokAll
	TokHas
	TokLParen
	TokRParen
	TokAnd
	TokOr
	TokEof
)

Types

type Token

type Token struct {
	Kind  tokenKind
	Value interface{}
}

func Tokenize

func Tokenize(input string) (tokens []Token, err error)