Versions in this module Expand all Collapse all v0 v0.1.3 Jan 21, 2023 Changes in this version + const ToTheEnd + var AddAssignToken = NewSpecialToken("+=") + var AddToken = NewSpecialToken("+") + var AddressToken = NewSpecialToken("&") + var AndToken = NewSpecialToken("&") + var AssertToken = NewReservedToken("assert") + var AssignToken = NewSpecialToken("=") + var BlockBeginToken = NewSpecialToken("{") + var BlockEndToken = NewSpecialToken("}") + var BoolToken = NewTypeToken("bool") + var BooleanAndToken = NewSpecialToken("&&") + var BooleanOrToken = NewSpecialToken("||") + var BreakToken = NewReservedToken("break") + var ByteToken = NewTypeToken("byte") + var CallToken = NewReservedToken("call") + var CaseToken = NewIdentifierToken("case") + var CatchToken = NewReservedToken("catch") + var ChanToken = NewTypeToken("chan") + var ChannelReceiveToken = NewSpecialToken("<-") + var ClearToken = NewIdentifierToken("clear") + var ColonToken = NewSpecialToken(":") + var CommaToken = NewSpecialToken(",") + var ConstToken = NewReservedToken("const") + var ContinueToken = NewReservedToken("continue") + var DataBeginToken = NewSpecialToken("{") + var DataEndToken = NewSpecialToken("}") + var DecrementToken = NewSpecialToken("--") + var DefaultToken = NewIdentifierToken("default") + var DeferToken = NewReservedToken("defer") + var DefineToken = NewSpecialToken(":=") + var DirectiveToken = NewSpecialToken("@") + var DivideAssignToken = NewSpecialToken("/=") + var DivideToken = NewSpecialToken("/") + var DotToken = NewSpecialToken(".") + var ElseToken = NewReservedToken("else") + var EmptyBlockToken = NewSpecialToken("{}") + var EmptyInitializerToken = NewSpecialToken("{}") + var EmptyInterfaceToken = NewTypeToken("interface{}") + var EmptyToken = NewSpecialToken("") + var EndOfArrayToken = NewSpecialToken("]") + var EndOfListToken = NewSpecialToken(")") + var EndOfTokens = Token + var EqualsToken = NewSpecialToken("==") + var ErrorToken = NewIdentifierToken("error") + var ExitToken = NewReservedToken("exit") + var ExponentToken = NewSpecialToken("^") + var ExtendedReservedWords = map[Token]bool + var FallthroughToken = NewReservedToken("fallthrough") + var Float32Token = NewTypeToken("float32") + var Float64Token = NewTypeToken("float64") + var ForToken = NewReservedToken("for") + var FuncToken = NewReservedToken("func") + var GoToken = NewReservedToken("go") + var GreaterThanOrEqualsToken = NewSpecialToken(">=") + var GreaterThanToken = NewSpecialToken(">") + var IfToken = NewReservedToken("if") + var ImportToken = NewReservedToken("import") + var IncrementToken = NewSpecialToken("++") + var Int32Token = NewTypeToken("int32") + var Int64Token = NewTypeToken("int64") + var IntToken = NewTypeToken("int") + var InterfaceToken = NewIdentifierToken("interface") + var LessThanOrEqualsToken = NewSpecialToken("<=") + var LessThanToken = NewSpecialToken("<") + var MakeToken = NewReservedToken("make") + var MapToken = NewTypeToken("map") + var ModuloToken = NewSpecialToken("%") + var MultiplyAssignToken = NewSpecialToken("*=") + var MultiplyToken = NewSpecialToken("*") + var NegateToken = NewSpecialToken("-") + var NilToken = NewReservedToken("nil") + var NotEqualsToken = NewSpecialToken("!=") + var NotToken = NewSpecialToken("!") + var OptionalToken = NewSpecialToken("?") + var OrToken = NewSpecialToken("|") + var PackageToken = NewReservedToken("package") + var PanicToken = NewReservedToken("panic") + var PointerToken = NewSpecialToken("*") + var PrintToken = NewReservedToken("print") + var RangeToken = NewIdentifierToken("range") + var ReservedWords = map[Token]bool + var ReturnToken = NewReservedToken("return") + var SemicolonToken = NewSpecialToken(";") + var ShiftLeftToken = NewSpecialToken("<<") + var ShiftRightToken = NewSpecialToken(">>") + var SpecialTokens = map[Token]bool + var StartOfArrayToken = NewSpecialToken("[") + var StartOfListToken = NewSpecialToken("(") + var StringToken = NewTypeToken("string") + var StructToken = NewTypeToken("struct") + var SubtractAssignToken = NewSpecialToken("-=") + var SubtractToken = NewSpecialToken("-") + var SwitchToken = NewReservedToken("switch") + var TestToken = NewIdentifierToken("test") + var TryToken = NewReservedToken("try") + var TypeToken = NewReservedToken("type") + var TypeTokens = map[Token]bool + var VarToken = NewReservedToken("var") + var VariadicToken = NewSpecialToken("...") + var WhenToken = NewIdentifierToken("when") + func InList(s Token, test ...Token) bool + func IsSymbol(s string) bool + type Token struct + func NewFloatToken(spelling string) Token + func NewIdentifierToken(spelling string) Token + func NewIntegerToken(spelling string) Token + func NewReservedToken(spelling string) Token + func NewSpecialToken(spelling string) Token + func NewStringToken(spelling string) Token + func NewToken(class TokenClass, spelling string) Token + func NewTypeToken(spelling string) Token + func NewValueToken(spelling string) Token + func (t Token) Boolean() bool + func (t Token) Float() float64 + func (t Token) Integer() int64 + func (t Token) IsClass(class TokenClass) bool + func (t Token) IsIdentifier() bool + func (t Token) IsName() bool + func (t Token) IsReserved(includeExtensions bool) bool + func (t Token) IsString() bool + func (t Token) IsValue() bool + func (t Token) Spelling() string + func (t Token) String() string + type TokenClass int + const BooleanTokenClass + const EndOfTokensClass + const FloatTokenClass + const IdentifierTokenClass + const IntegerTokenClass + const ReservedTokenClass + const SpecialTokenClass + const StringTokenClass + const TypeTokenClass + const ValueTokenClass + func (c TokenClass) String() string + type Tokenizer struct + Line []int + Pos []int + Source []string + TokenP int + Tokens []Token + func New(src string, isCode bool) *Tokenizer + func (t *Tokenizer) Advance(p int) + func (t *Tokenizer) AnyNext(test ...Token) bool + func (t *Tokenizer) AtEnd() bool + func (t *Tokenizer) DumpTokens(before, after int) + func (t *Tokenizer) GetLine(line int) string + func (t *Tokenizer) GetSource() string + func (t *Tokenizer) GetTokens(pos1, pos2 int, spacing bool) string + func (t *Tokenizer) IsNext(test Token) bool + func (t *Tokenizer) Mark() int + func (t *Tokenizer) Next() Token + func (t *Tokenizer) NextText() string + func (t *Tokenizer) Peek(offset int) Token + func (t *Tokenizer) PeekText(offset int) string + func (t *Tokenizer) Remainder() string + func (t *Tokenizer) Reset() + func (t *Tokenizer) Set(mark int) + func (t *Tokenizer) SetLineNumber(line int) error