token

Documentation

Index

• Variables
• type Mode
• type Token
  • func ComsumeRunesAny(inputs []rune, current int, runeType Type, expectRunes ...rune) (token Token, next int)
  • func ComsumeStringsAny(inputs []rune, current int, runeType Type, mode Mode, expectStrs ...string) (token Token, next int)
  • func ConsumeIdentifier(inputs []rune, current int, runeType Type) (token Token, next int)
  • func Tokenizer(inputs []rune, f func(inputs []rune, current int) (token Token, next int), ...) []Token
• type TokenizerFunc
• type Type

Variables

var (
	Literals = map[string]struct{}{"<": {}, ">": {}}
)

Types

type Mode

type Mode uint

A mode value is a set of flags (or 0). They control scanner behavior.

const (
	ModeCaseSensitive Mode = 1 << iota
	ModeRegexpPerl
	ModeRegexpPosix
)

type Token

type Token struct {
	Typ   Type
	Value string
}

func ComsumeRunesAny

func ComsumeRunesAny(inputs []rune, current int, runeType Type, expectRunes ...rune) (token Token, next int)

func ComsumeStringsAny

func ComsumeStringsAny(inputs []rune, current int, runeType Type, mode Mode, expectStrs ...string) (token Token, next int)

func ConsumeIdentifier

func ConsumeIdentifier(inputs []rune, current int, runeType Type) (token Token, next int)

func Tokenizer

func Tokenizer(inputs []rune, f func(inputs []rune, current int) (token Token, next int), strict bool) []Token

type <value>, type <value>

type TokenizerFunc

type TokenizerFunc interface {
	IsLiteral() bool
	IsOperator() bool
	IsKeyword() bool
}

type Type

type Type int

Token is the set of lexical tokens of the Go programming language.

const (
	// Special tokens
	TypeILLEGAL Type = iota
	TypeEOF
	TypeCOMMENT
	TypeIgnored

	TypeLiteral
	TypeOperator
	TypeKeyword
)