lexer

Documentation

Index

• func DecodeUTF8Rune(s string) (rune, int)
• func IsIdentifier(text string) bool
• func IsIdentifierContinue(codePoint rune) bool
• func IsIdentifierStart(codePoint rune) bool
• func Keywords() map[string]T
• func NumberToMinifiedName(i int) string
• func RangeOfIdentifier(source logging.Source, loc ast.Loc) ast.Range
• func StringToUTF16(text string) []uint16
• func UTF16ToString(text []uint16) string
• type Lexer
  • func NewLexer(log logging.Log, source logging.Source) Lexer
  • func (lexer *Lexer) Expect(token T)
  • func (lexer *Lexer) ExpectContextualKeyword(text string)
  • func (lexer *Lexer) ExpectInsideJSXElement(token T)
  • func (lexer *Lexer) ExpectJSXElementChild(token T)
  • func (lexer *Lexer) ExpectOrInsertSemicolon()
  • func (lexer *Lexer) Expected(token T)
  • func (lexer *Lexer) ExpectedString(text string)
  • func (lexer *Lexer) IsContextualKeyword(text string) bool
  • func (lexer *Lexer) IsIdentifierOrKeyword() bool
  • func (lexer *Lexer) Loc() ast.Loc
  • func (lexer *Lexer) Next()
  • func (lexer *Lexer) NextInsideJSXElement()
  • func (lexer *Lexer) NextJSXElementChild()
  • func (lexer *Lexer) Range() ast.Range
  • func (lexer *Lexer) Raw() string
  • func (lexer *Lexer) RawTemplateContents() string
  • func (lexer *Lexer) RescanCloseBraceAsTemplateToken()
  • func (lexer *Lexer) ScanRegExp()
  • func (lexer *Lexer) SyntaxError()
  • func (lexer *Lexer) Unexpected()
• type LexerPanic
• type T

Functions

func DecodeUTF8Rune

func DecodeUTF8Rune(s string) (rune, int)

This is a clone of "utf8.DecodeRuneInString" that has been modified to encode using WTF-8 instead. See https://simonsapin.github.io/wtf-8/ for more info.

func IsIdentifier

func IsIdentifier(text string) bool

func IsIdentifierContinue

func IsIdentifierContinue(codePoint rune) bool

func IsIdentifierStart

func IsIdentifierStart(codePoint rune) bool

func Keywords

func Keywords() map[string]T

func NumberToMinifiedName

func NumberToMinifiedName(i int) string

func RangeOfIdentifier

func RangeOfIdentifier(source logging.Source, loc ast.Loc) ast.Range

func StringToUTF16

func StringToUTF16(text string) []uint16

func UTF16ToString

func UTF16ToString(text []uint16) string

Types

type Lexer

type Lexer struct {
	Token            T
	HasNewlineBefore bool

	StringLiteral []uint16
	Identifier    string
	Number        float64
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(log logging.Log, source logging.Source) Lexer

func (*Lexer) Expect

func (lexer *Lexer) Expect(token T)

func (*Lexer) ExpectContextualKeyword

func (lexer *Lexer) ExpectContextualKeyword(text string)

func (*Lexer) ExpectInsideJSXElement

func (lexer *Lexer) ExpectInsideJSXElement(token T)

func (*Lexer) ExpectJSXElementChild

func (lexer *Lexer) ExpectJSXElementChild(token T)

func (*Lexer) ExpectOrInsertSemicolon

func (lexer *Lexer) ExpectOrInsertSemicolon()

func (*Lexer) Expected

func (lexer *Lexer) Expected(token T)

func (*Lexer) ExpectedString

func (lexer *Lexer) ExpectedString(text string)

func (*Lexer) IsContextualKeyword

func (lexer *Lexer) IsContextualKeyword(text string) bool

func (*Lexer) IsIdentifierOrKeyword

func (lexer *Lexer) IsIdentifierOrKeyword() bool

func (*Lexer) Loc

func (lexer *Lexer) Loc() ast.Loc

func (*Lexer) Next

func (lexer *Lexer) Next()

func (*Lexer) NextInsideJSXElement

func (lexer *Lexer) NextInsideJSXElement()

func (*Lexer) NextJSXElementChild

func (lexer *Lexer) NextJSXElementChild()

func (*Lexer) Range

func (lexer *Lexer) Range() ast.Range

func (*Lexer) Raw

func (lexer *Lexer) Raw() string

func (*Lexer) RawTemplateContents

func (lexer *Lexer) RawTemplateContents() string

func (*Lexer) RescanCloseBraceAsTemplateToken

func (lexer *Lexer) RescanCloseBraceAsTemplateToken()

func (*Lexer) ScanRegExp

func (lexer *Lexer) ScanRegExp()

func (*Lexer) SyntaxError

func (lexer *Lexer) SyntaxError()

func (*Lexer) Unexpected

func (lexer *Lexer) Unexpected()

type LexerPanic

type LexerPanic struct{}

type T

type T uint

const (
	TEndOfFile T = iota
	TSyntaxError

	// Literals
	TNoSubstitutionTemplateLiteral // Contents are in lexer.StringLiteral ([]uint16)
	TNumericLiteral                // Contents are in lexer.Number (float64)
	TStringLiteral                 // Contents are in lexer.StringLiteral ([]uint16)
	TBigIntegerLiteral             // Contents are in lexer.Identifier (string)

	// Pseudo-literals
	TTemplateHead   // Contents are in lexer.StringLiteral ([]uint16)
	TTemplateMiddle // Contents are in lexer.StringLiteral ([]uint16)
	TTemplateTail   // Contents are in lexer.StringLiteral ([]uint16)

	// Punctuation
	TAmpersand
	TAmpersandAmpersand
	TAsterisk
	TAsteriskAsterisk
	TAt
	TBar
	TBarBar
	TCaret
	TCloseBrace
	TCloseBracket
	TCloseParen
	TColon
	TComma
	TDot
	TDotDotDot
	TEqualsEquals
	TEqualsEqualsEquals
	TEqualsGreaterThan
	TExclamation
	TExclamationEquals
	TExclamationEqualsEquals
	TGreaterThan
	TGreaterThanEquals
	TGreaterThanGreaterThan
	TGreaterThanGreaterThanGreaterThan
	TLessThan
	TLessThanEquals
	TLessThanLessThan
	TMinus
	TMinusMinus
	TOpenBrace
	TOpenBracket
	TOpenParen
	TPercent
	TPlus
	TPlusPlus
	TQuestion
	TQuestionDot
	TQuestionQuestion
	TSemicolon
	TSlash
	TTilde

	// Assignments
	TAmpersandEquals
	TAsteriskAsteriskEquals
	TAsteriskEquals
	TBarEquals
	TCaretEquals
	TEquals
	TGreaterThanGreaterThanEquals
	TGreaterThanGreaterThanGreaterThanEquals
	TLessThanLessThanEquals
	TMinusEquals
	TPercentEquals
	TPlusEquals
	TSlashEquals

	// Identifiers
	TIdentifier     // Contents are in lexer.Identifier (string)
	TEscapedKeyword // A keyword that has been escaped as an identifer

	// Reserved words
	TBreak
	TCase
	TCatch
	TClass
	TConst
	TContinue
	TDebugger
	TDefault
	TDelete
	TDo
	TElse
	TEnum
	TExport
	TExtends
	TFalse
	TFinally
	TFor
	TFunction
	TIf
	TImport
	TIn
	TInstanceOf
	TNew
	TNull
	TReturn
	TSuper
	TSwitch
	TThis
	TThrow
	TTrue
	TTry
	TTypeOf
	TVar
	TVoid
	TWhile
	TWith

	// Strict mode reserved words
	TImplements
	TInterface
	TLet
	TPackage
	TPrivate
	TProtected
	TPublic
	TStatic
	TYield
)

If you add a new token, remember to add it to "tokenToString" too