js_lexer

Documentation

Overview

This file was automatically generated by gen-unicode-table.js. Do not edit.

Index

• Variables
• func AppendUTF16ToBytes(bytes []byte, text []uint16) []byte
• func DecodeWTF8Rune(s string) (rune, int)
• func ForceValidIdentifier(text string) string
• func IsIdentifier(text string) bool
• func IsIdentifierContinue(codePoint rune) bool
• func IsIdentifierStart(codePoint rune) bool
• func IsIdentifierUTF16(text []uint16) bool
• func IsWhitespace(codePoint rune) bool
• func RangeOfIdentifier(source logger.Source, loc logger.Loc) logger.Range
• func StringToUTF16(text string) []uint16
• func UTF16EqualsString(text []uint16, str string) bool
• func UTF16EqualsUTF16(a []uint16, b []uint16) bool
• func UTF16ToString(text []uint16) string
• type Lexer
  • func NewLexer(log logger.Log, source logger.Source) Lexer
  • func NewLexerJSON(log logger.Log, source logger.Source, allowComments bool) Lexer
  • func (lexer *Lexer) Expect(token T)
  • func (lexer *Lexer) ExpectContextualKeyword(text string)
  • func (lexer *Lexer) ExpectGreaterThan(isInsideJSXElement bool)
  • func (lexer *Lexer) ExpectInsideJSXElement(token T)
  • func (lexer *Lexer) ExpectJSXElementChild(token T)
  • func (lexer *Lexer) ExpectLessThan(isInsideJSXElement bool)
  • func (lexer *Lexer) ExpectOrInsertSemicolon()
  • func (lexer *Lexer) Expected(token T)
  • func (lexer *Lexer) ExpectedString(text string)
  • func (lexer *Lexer) IsContextualKeyword(text string) bool
  • func (lexer *Lexer) IsIdentifierOrKeyword() bool
  • func (lexer *Lexer) Loc() logger.Loc
  • func (lexer *Lexer) Next()
  • func (lexer *Lexer) NextInsideJSXElement()
  • func (lexer *Lexer) NextJSXElementChild()
  • func (lexer *Lexer) Range() logger.Range
  • func (lexer *Lexer) Raw() string
  • func (lexer *Lexer) RawTemplateContents() string
  • func (lexer *Lexer) RescanCloseBraceAsTemplateToken()
  • func (lexer *Lexer) ScanRegExp()
  • func (lexer *Lexer) SyntaxError()
  • func (lexer *Lexer) Unexpected()
• type LexerPanic
• type T

Variables

var Keywords = map[string]T{

	"break":      TBreak,
	"case":       TCase,
	"catch":      TCatch,
	"class":      TClass,
	"const":      TConst,
	"continue":   TContinue,
	"debugger":   TDebugger,
	"default":    TDefault,
	"delete":     TDelete,
	"do":         TDo,
	"else":       TElse,
	"enum":       TEnum,
	"export":     TExport,
	"extends":    TExtends,
	"false":      TFalse,
	"finally":    TFinally,
	"for":        TFor,
	"function":   TFunction,
	"if":         TIf,
	"import":     TImport,
	"in":         TIn,
	"instanceof": TInstanceof,
	"new":        TNew,
	"null":       TNull,
	"return":     TReturn,
	"super":      TSuper,
	"switch":     TSwitch,
	"this":       TThis,
	"throw":      TThrow,
	"true":       TTrue,
	"try":        TTry,
	"typeof":     TTypeof,
	"var":        TVar,
	"void":       TVoid,
	"while":      TWhile,
	"with":       TWith,
}

var StrictModeReservedWords = map[string]bool{
	"implements": true,
	"interface":  true,
	"let":        true,
	"package":    true,
	"private":    true,
	"protected":  true,
	"public":     true,
	"static":     true,
	"yield":      true,
}

Functions

func AppendUTF16ToBytes

func AppendUTF16ToBytes(bytes []byte, text []uint16) []byte

Does "append(bytes, UTF16ToString(text))" without a temporary allocation

func DecodeWTF8Rune

func DecodeWTF8Rune(s string) (rune, int)

This is a clone of "utf8.DecodeRuneInString" that has been modified to decode using WTF-8 instead. See https://simonsapin.github.io/wtf-8/ for more info.

func ForceValidIdentifier

func ForceValidIdentifier(text string) string

func IsIdentifier

func IsIdentifier(text string) bool

func IsIdentifierContinue

func IsIdentifierContinue(codePoint rune) bool

func IsIdentifierStart

func IsIdentifierStart(codePoint rune) bool

func IsIdentifierUTF16

func IsIdentifierUTF16(text []uint16) bool

This does "IsIdentifier(UTF16ToString(text))" without any allocations

func IsWhitespace

func IsWhitespace(codePoint rune) bool

See the "White Space Code Points" table in the ECMAScript standard

func RangeOfIdentifier

func RangeOfIdentifier(source logger.Source, loc logger.Loc) logger.Range

func StringToUTF16

func StringToUTF16(text string) []uint16

func UTF16EqualsString

func UTF16EqualsString(text []uint16, str string) bool

Does "UTF16ToString(text) == str" without a temporary allocation

func UTF16EqualsUTF16

func UTF16EqualsUTF16(a []uint16, b []uint16) bool

func UTF16ToString

func UTF16ToString(text []uint16) string

Types

type Lexer

type Lexer struct {
	ApproximateNewlineCount   int
	Token                     T
	HasNewlineBefore          bool
	HasPureCommentBefore      bool
	PreserveAllCommentsBefore bool
	CommentsToPreserveBefore  []js_ast.Comment

	StringLiteral            []uint16
	Identifier               string
	JSXFactoryPragmaComment  js_ast.Span
	JSXFragmentPragmaComment js_ast.Span
	SourceMappingURL         js_ast.Span
	Number                   float64

	// The log is disabled during speculative scans that may backtrack
	IsLogDisabled bool
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(log logger.Log, source logger.Source) Lexer

func NewLexerJSON

func NewLexerJSON(log logger.Log, source logger.Source, allowComments bool) Lexer

func (*Lexer) Expect

func (lexer *Lexer) Expect(token T)

func (*Lexer) ExpectContextualKeyword

func (lexer *Lexer) ExpectContextualKeyword(text string)

func (*Lexer) ExpectGreaterThan

func (lexer *Lexer) ExpectGreaterThan(isInsideJSXElement bool)

This parses a single ">" token. If that is the first part of a longer token, this function splits off the first ">" and leaves the remainder of the current token as another, smaller token. For example, ">>=" becomes ">=".

func (*Lexer) ExpectInsideJSXElement

func (lexer *Lexer) ExpectInsideJSXElement(token T)

func (*Lexer) ExpectJSXElementChild

func (lexer *Lexer) ExpectJSXElementChild(token T)

func (*Lexer) ExpectLessThan

func (lexer *Lexer) ExpectLessThan(isInsideJSXElement bool)

This parses a single "<" token. If that is the first part of a longer token, this function splits off the first "<" and leaves the remainder of the current token as another, smaller token. For example, "<<=" becomes "<=".

func (*Lexer) ExpectOrInsertSemicolon

func (lexer *Lexer) ExpectOrInsertSemicolon()

func (*Lexer) Expected

func (lexer *Lexer) Expected(token T)

func (*Lexer) ExpectedString

func (lexer *Lexer) ExpectedString(text string)

func (*Lexer) IsContextualKeyword

func (lexer *Lexer) IsContextualKeyword(text string) bool

func (*Lexer) IsIdentifierOrKeyword

func (lexer *Lexer) IsIdentifierOrKeyword() bool

func (*Lexer) Loc

func (lexer *Lexer) Loc() logger.Loc

func (*Lexer) Next

func (lexer *Lexer) Next()

func (*Lexer) NextInsideJSXElement

func (lexer *Lexer) NextInsideJSXElement()

func (*Lexer) NextJSXElementChild

func (lexer *Lexer) NextJSXElementChild()

func (*Lexer) Range

func (lexer *Lexer) Range() logger.Range

func (*Lexer) Raw

func (lexer *Lexer) Raw() string

func (*Lexer) RawTemplateContents

func (lexer *Lexer) RawTemplateContents() string

func (*Lexer) RescanCloseBraceAsTemplateToken

func (lexer *Lexer) RescanCloseBraceAsTemplateToken()

func (*Lexer) ScanRegExp

func (lexer *Lexer) ScanRegExp()

func (*Lexer) SyntaxError

func (lexer *Lexer) SyntaxError()

func (*Lexer) Unexpected

func (lexer *Lexer) Unexpected()

type LexerPanic

type LexerPanic struct{}

type T

type T uint

const (
	TEndOfFile T = iota
	TSyntaxError

	// "#!/usr/bin/env node"
	THashbang

	// Literals
	TNoSubstitutionTemplateLiteral // Contents are in lexer.StringLiteral ([]uint16)
	TNumericLiteral                // Contents are in lexer.Number (float64)
	TStringLiteral                 // Contents are in lexer.StringLiteral ([]uint16)
	TBigIntegerLiteral             // Contents are in lexer.Identifier (string)

	// Pseudo-literals
	TTemplateHead   // Contents are in lexer.StringLiteral ([]uint16)
	TTemplateMiddle // Contents are in lexer.StringLiteral ([]uint16)
	TTemplateTail   // Contents are in lexer.StringLiteral ([]uint16)

	// Punctuation
	TAmpersand
	TAmpersandAmpersand
	TAsterisk
	TAsteriskAsterisk
	TAt
	TBar
	TBarBar
	TCaret
	TCloseBrace
	TCloseBracket
	TCloseParen
	TColon
	TComma
	TDot
	TDotDotDot
	TEqualsEquals
	TEqualsEqualsEquals
	TEqualsGreaterThan
	TExclamation
	TExclamationEquals
	TExclamationEqualsEquals
	TGreaterThan
	TGreaterThanEquals
	TGreaterThanGreaterThan
	TGreaterThanGreaterThanGreaterThan
	TLessThan
	TLessThanEquals
	TLessThanLessThan
	TMinus
	TMinusMinus
	TOpenBrace
	TOpenBracket
	TOpenParen
	TPercent
	TPlus
	TPlusPlus
	TQuestion
	TQuestionDot
	TQuestionQuestion
	TSemicolon
	TSlash
	TTilde

	// Assignments
	TAmpersandAmpersandEquals
	TAmpersandEquals
	TAsteriskAsteriskEquals
	TAsteriskEquals
	TBarBarEquals
	TBarEquals
	TCaretEquals
	TEquals
	TGreaterThanGreaterThanEquals
	TGreaterThanGreaterThanGreaterThanEquals
	TLessThanLessThanEquals
	TMinusEquals
	TPercentEquals
	TPlusEquals
	TQuestionQuestionEquals
	TSlashEquals

	// Class-private fields and methods
	TPrivateIdentifier

	// Identifiers
	TIdentifier     // Contents are in lexer.Identifier (string)
	TEscapedKeyword // A keyword that has been escaped as an identifer

	// Reserved words
	TBreak
	TCase
	TCatch
	TClass
	TConst
	TContinue
	TDebugger
	TDefault
	TDelete
	TDo
	TElse
	TEnum
	TExport
	TExtends
	TFalse
	TFinally
	TFor
	TFunction
	TIf
	TImport
	TIn
	TInstanceof
	TNew
	TNull
	TReturn
	TSuper
	TSwitch
	TThis
	TThrow
	TTrue
	TTry
	TTypeof
	TVar
	TVoid
	TWhile
	TWith
)

If you add a new token, remember to add it to "tokenToString" too