js_lexer

Documentation

Index

• Variables
• func RangeOfIdentifier(source logger.Source, loc logger.Loc) logger.Range
• func RangeOfImportAssertOrWith(source logger.Source, assertOrWith ast.AssertOrWithEntry, which KeyOrValue) logger.Range
• type CommentBefore
• type JSONFlavor
• type KeyOrValue
• type Lexer
  • func NewLexer(log logger.Log, source logger.Source, ts config.TSOptions) Lexer
  • func NewLexerGlobalName(log logger.Log, source logger.Source) Lexer
  • func NewLexerJSON(log logger.Log, source logger.Source, json JSONFlavor, errorSuffix string) Lexer
  • func (lexer *Lexer) AddRangeErrorWithNotes(r logger.Range, text string, notes []logger.MsgData)
  • func (lexer *Lexer) CookedAndRawTemplateContents() ([]uint16, string)
  • func (lexer *Lexer) Expect(token T)
  • func (lexer *Lexer) ExpectContextualKeyword(text string)
  • func (lexer *Lexer) ExpectGreaterThan(isInsideJSXElement bool)
  • func (lexer *Lexer) ExpectInsideJSXElement(token T)
  • func (lexer *Lexer) ExpectJSXElementChild(token T)
  • func (lexer *Lexer) ExpectLessThan(isInsideJSXElement bool)
  • func (lexer *Lexer) ExpectOrInsertSemicolon()
  • func (lexer *Lexer) Expected(token T)
  • func (lexer *Lexer) ExpectedString(text string)
  • func (lexer *Lexer) IsContextualKeyword(text string) bool
  • func (lexer *Lexer) IsIdentifierOrKeyword() bool
  • func (lexer *Lexer) Loc() logger.Loc
  • func (lexer *Lexer) Next()
  • func (lexer *Lexer) NextInsideJSXElement()
  • func (lexer *Lexer) NextJSXElementChild()
  • func (lexer *Lexer) Range() logger.Range
  • func (lexer *Lexer) Raw() string
  • func (lexer *Lexer) RescanCloseBraceAsTemplateToken()
  • func (lexer *Lexer) ScanRegExp()
  • func (lexer *Lexer) StringLiteral() []uint16
  • func (lexer *Lexer) SyntaxError()
  • func (lexer *Lexer) Unexpected()
• type LexerPanic
• type MaybeSubstring
• type T
  • func (t T) IsAssign() bool

Variables

var Keywords = map[string]T{

	"break":      TBreak,
	"case":       TCase,
	"catch":      TCatch,
	"class":      TClass,
	"const":      TConst,
	"continue":   TContinue,
	"debugger":   TDebugger,
	"default":    TDefault,
	"delete":     TDelete,
	"do":         TDo,
	"else":       TElse,
	"enum":       TEnum,
	"export":     TExport,
	"extends":    TExtends,
	"false":      TFalse,
	"finally":    TFinally,
	"for":        TFor,
	"function":   TFunction,
	"if":         TIf,
	"import":     TImport,
	"in":         TIn,
	"instanceof": TInstanceof,
	"new":        TNew,
	"null":       TNull,
	"return":     TReturn,
	"super":      TSuper,
	"switch":     TSwitch,
	"this":       TThis,
	"throw":      TThrow,
	"true":       TTrue,
	"try":        TTry,
	"typeof":     TTypeof,
	"var":        TVar,
	"void":       TVoid,
	"while":      TWhile,
	"with":       TWith,
}

var StrictModeReservedWords = map[string]bool{
	"implements": true,
	"interface":  true,
	"let":        true,
	"package":    true,
	"private":    true,
	"protected":  true,
	"public":     true,
	"static":     true,
	"yield":      true,
}

Functions

func RangeOfIdentifier

func RangeOfIdentifier(source logger.Source, loc logger.Loc) logger.Range

func RangeOfImportAssertOrWith

func RangeOfImportAssertOrWith(source logger.Source, assertOrWith ast.AssertOrWithEntry, which KeyOrValue) logger.Range

Types

type CommentBefore

type CommentBefore uint8

const (
	PureCommentBefore CommentBefore = 1 << iota
	KeyCommentBefore
	NoSideEffectsCommentBefore
)

type JSONFlavor

type JSONFlavor uint8

const (
	// Specification: https://json.org/
	JSON JSONFlavor = iota

	// TypeScript's JSON superset is not documented but appears to allow:
	// - Comments: https://github.com/microsoft/TypeScript/issues/4987
	// - Trailing commas
	// - Full JS number syntax
	TSConfigJSON

	// This is used by the JavaScript lexer
	NotJSON
)

type KeyOrValue

type KeyOrValue uint8

const (
	KeyRange KeyOrValue = iota
	ValueRange
	KeyAndValueRange
)

type Lexer

type Lexer struct {
	LegalCommentsBeforeToken []logger.Range
	CommentsBeforeToken      []logger.Range
	AllComments              []logger.Range
	Identifier               MaybeSubstring

	JSXFactoryPragmaComment      logger.Span
	JSXFragmentPragmaComment     logger.Span
	JSXRuntimePragmaComment      logger.Span
	JSXImportSourcePragmaComment logger.Span
	SourceMappingURL             logger.Span
	BadArrowInTSXSuggestion      string

	Number float64

	ApproximateNewlineCount     int
	CouldBeBadArrowInTSX        int
	BadArrowInTSXRange          logger.Range
	LegacyOctalLoc              logger.Loc
	AwaitKeywordLoc             logger.Loc
	FnOrArrowStartLoc           logger.Loc
	PreviousBackslashQuoteInJSX logger.Range
	LegacyHTMLCommentRange      logger.Range

	Token T

	HasNewlineBefore         bool
	HasCommentBefore         CommentBefore
	IsLegacyOctalLiteral     bool
	PrevTokenWasAwaitKeyword bool

	// The log is disabled during speculative scans that may backtrack
	IsLogDisabled bool
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(log logger.Log, source logger.Source, ts config.TSOptions) Lexer

func NewLexerGlobalName

func NewLexerGlobalName(log logger.Log, source logger.Source) Lexer

func NewLexerJSON

func NewLexerJSON(log logger.Log, source logger.Source, json JSONFlavor, errorSuffix string) Lexer

func (*Lexer) AddRangeErrorWithNotes

func (lexer *Lexer) AddRangeErrorWithNotes(r logger.Range, text string, notes []logger.MsgData)

func (*Lexer) CookedAndRawTemplateContents

func (lexer *Lexer) CookedAndRawTemplateContents() ([]uint16, string)

func (*Lexer) Expect

func (lexer *Lexer) Expect(token T)

func (*Lexer) ExpectContextualKeyword

func (lexer *Lexer) ExpectContextualKeyword(text string)

func (*Lexer) ExpectGreaterThan

func (lexer *Lexer) ExpectGreaterThan(isInsideJSXElement bool)

This parses a single ">" token. If that is the first part of a longer token, this function splits off the first ">" and leaves the remainder of the current token as another, smaller token. For example, ">>=" becomes ">=".

func (*Lexer) ExpectInsideJSXElement

func (lexer *Lexer) ExpectInsideJSXElement(token T)

func (*Lexer) ExpectJSXElementChild

func (lexer *Lexer) ExpectJSXElementChild(token T)

func (*Lexer) ExpectLessThan

func (lexer *Lexer) ExpectLessThan(isInsideJSXElement bool)

This parses a single "<" token. If that is the first part of a longer token, this function splits off the first "<" and leaves the remainder of the current token as another, smaller token. For example, "<<=" becomes "<=".

func (*Lexer) ExpectOrInsertSemicolon

func (lexer *Lexer) ExpectOrInsertSemicolon()

func (*Lexer) Expected

func (lexer *Lexer) Expected(token T)

func (*Lexer) ExpectedString

func (lexer *Lexer) ExpectedString(text string)

func (*Lexer) IsContextualKeyword

func (lexer *Lexer) IsContextualKeyword(text string) bool

func (*Lexer) IsIdentifierOrKeyword

func (lexer *Lexer) IsIdentifierOrKeyword() bool

func (*Lexer) Loc

func (lexer *Lexer) Loc() logger.Loc

func (*Lexer) Next

func (lexer *Lexer) Next()

func (*Lexer) NextInsideJSXElement

func (lexer *Lexer) NextInsideJSXElement()

func (*Lexer) NextJSXElementChild

func (lexer *Lexer) NextJSXElementChild()

func (*Lexer) Range

func (lexer *Lexer) Range() logger.Range

func (*Lexer) Raw

func (lexer *Lexer) Raw() string

func (*Lexer) RescanCloseBraceAsTemplateToken

func (lexer *Lexer) RescanCloseBraceAsTemplateToken()

func (*Lexer) ScanRegExp

func (lexer *Lexer) ScanRegExp()

func (*Lexer) StringLiteral

func (lexer *Lexer) StringLiteral() []uint16

func (*Lexer) SyntaxError

func (lexer *Lexer) SyntaxError()

func (*Lexer) Unexpected

func (lexer *Lexer) Unexpected()

type LexerPanic

type LexerPanic struct{}

type MaybeSubstring

type MaybeSubstring struct {
	String string
	Start  ast.Index32
}

This represents a string that is maybe a substring of the current file's "source.Contents" string. The point of doing this is that if it is a substring (the common case), then we can represent it more efficiently.

For compactness and performance, the JS AST represents identifiers as a symbol reference instead of as a string. However, we need to track the string between the first pass and the second pass because the string is only resolved to a symbol in the second pass. To avoid allocating extra memory to store the string, we instead use an index+length slice of the original JS source code. That index is what "Start" represents here. The length is just "len(String)".

Set "Start" to invalid (the zero value) if "String" is not a substring of "source.Contents". This is the case for escaped identifiers. For example, the identifier "fo\u006f" would be "MaybeSubstring{String: "foo"}". It's critical that any code changing the "String" also set "Start" to the zero value, which is best done by just overwriting the whole "MaybeSubstring".

The substring range used to be recovered automatically from the string but that relied on the Go "unsafe" package which can hypothetically break under certain Go compiler optimization passes, so it has been removed and replaced with this more error-prone approach that doesn't use "unsafe".

type T

type T uint8

const (
	TEndOfFile T = iota
	TSyntaxError

	// "#!/usr/bin/env node"
	THashbang

	// Literals
	TNoSubstitutionTemplateLiteral // Contents are in lexer.StringLiteral ([]uint16)
	TNumericLiteral                // Contents are in lexer.Number (float64)
	TStringLiteral                 // Contents are in lexer.StringLiteral ([]uint16)
	TBigIntegerLiteral             // Contents are in lexer.Identifier (string)

	// Pseudo-literals
	TTemplateHead   // Contents are in lexer.StringLiteral ([]uint16)
	TTemplateMiddle // Contents are in lexer.StringLiteral ([]uint16)
	TTemplateTail   // Contents are in lexer.StringLiteral ([]uint16)

	// Punctuation
	TAmpersand
	TAmpersandAmpersand
	TAsterisk
	TAsteriskAsterisk
	TAt
	TBar
	TBarBar
	TCaret
	TCloseBrace
	TCloseBracket
	TCloseParen
	TColon
	TComma
	TDot
	TDotDotDot
	TEqualsEquals
	TEqualsEqualsEquals
	TEqualsGreaterThan
	TExclamation
	TExclamationEquals
	TExclamationEqualsEquals
	TGreaterThan
	TGreaterThanEquals
	TGreaterThanGreaterThan
	TGreaterThanGreaterThanGreaterThan
	TLessThan
	TLessThanEquals
	TLessThanLessThan
	TMinus
	TMinusMinus
	TOpenBrace
	TOpenBracket
	TOpenParen
	TPercent
	TPlus
	TPlusPlus
	TQuestion
	TQuestionDot
	TQuestionQuestion
	TSemicolon
	TSlash
	TTilde

	// Assignments (keep in sync with IsAssign() below)
	TAmpersandAmpersandEquals
	TAmpersandEquals
	TAsteriskAsteriskEquals
	TAsteriskEquals
	TBarBarEquals
	TBarEquals
	TCaretEquals
	TEquals
	TGreaterThanGreaterThanEquals
	TGreaterThanGreaterThanGreaterThanEquals
	TLessThanLessThanEquals
	TMinusEquals
	TPercentEquals
	TPlusEquals
	TQuestionQuestionEquals
	TSlashEquals

	// Class-private fields and methods
	TPrivateIdentifier

	// Identifiers
	TIdentifier     // Contents are in lexer.Identifier (string)
	TEscapedKeyword // A keyword that has been escaped as an identifer

	// Reserved words
	TBreak
	TCase
	TCatch
	TClass
	TConst
	TContinue
	TDebugger
	TDefault
	TDelete
	TDo
	TElse
	TEnum
	TExport
	TExtends
	TFalse
	TFinally
	TFor
	TFunction
	TIf
	TImport
	TIn
	TInstanceof
	TNew
	TNull
	TReturn
	TSuper
	TSwitch
	TThis
	TThrow
	TTrue
	TTry
	TTypeof
	TVar
	TVoid
	TWhile
	TWith
)

If you add a new token, remember to add it to "tokenToString" too

func (T) IsAssign

func (t T) IsAssign() bool