lexer

Documentation

Index

• Constants
• type Lexer
  • func NewLexer(source []byte, modeStack []LexerMode, offset int) *Lexer
  • func (s *Lexer) GetTokenValue(t *Token) []rune
  • func (s *Lexer) Lex() *Token
  • func (s *Lexer) ModeStack() []LexerMode
• type LexerMode
  • func (mode LexerMode) String() string
• type Pool
  • func NewPool(blockSize int) *Pool
  • func (p *Pool) Get() *Token
• type Token
  • func NewToken(pool *Pool, tokenType TokenType, offset int, length int) *Token
  • func (token Token) AstNode()
  • func (token Token) String() string
• type TokenType
  • func (tokenType *TokenType) MarshalJSON() ([]byte, error)
  • func (i TokenType) String() string

Constants

const DefaultBlockSize = 1024

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(source []byte, modeStack []LexerMode, offset int) *Lexer

func (*Lexer) GetTokenValue

func (s *Lexer) GetTokenValue(t *Token) []rune

GetTokenValue returns the []rune of the token source code

func (*Lexer) Lex

func (s *Lexer) Lex() *Token

Lex runs the lexing and returns a token

func (*Lexer) ModeStack

func (s *Lexer) ModeStack() []LexerMode

ModeStack returns a copy of modeStack

type LexerMode

type LexerMode uint8

const (
	ModeInitial LexerMode = iota
	ModeScripting
	ModeLookingForProperty
	ModeDoubleQuotes
	ModeNowDoc
	ModeHereDoc
	ModeEndHereDoc
	ModeBacktick
	ModeVarOffset
	ModeLookingForVarName
	ModeDocumentBlock
)

func (LexerMode) String

func (mode LexerMode) String() string

type Pool

type Pool struct {
	// contains filtered or unexported fields
}

func NewPool

func NewPool(blockSize int) *Pool

func (*Pool) Get

func (p *Pool) Get() *Token

type Token

type Token struct {
	Type   TokenType `json:"TokenType"`
	Offset int       `json:"Offset"`
	Length int       `json:"Length"`
}

func NewToken

func NewToken(pool *Pool, tokenType TokenType, offset int, length int) *Token

func (Token) AstNode

func (token Token) AstNode()

AstNode is a boilerplate for extending interface

func (Token) String

func (token Token) String() string

type TokenType

type TokenType uint8

const (
	// Misc
	Undefined TokenType = iota
	Unknown
	EndOfFile

	Abstract
	Array
	As
	Break
	Callable
	Case
	Catch
	Class
	ClassConstant
	Clone
	Const
	Continue
	Declare
	Default
	Do
	Echo
	Else
	ElseIf
	Empty
	EndDeclare
	EndFor
	EndForeach
	EndIf
	EndSwitch
	EndWhile
	EndHeredoc
	Eval
	Exit
	Extends
	Final
	Finally
	For
	ForEach
	Function
	Fn
	Global
	Goto
	HaltCompiler
	If
	Implements
	Include
	IncludeOnce
	InstanceOf
	InsteadOf
	Interface
	Isset
	List
	And
	Or
	Xor
	Namespace
	New
	Print
	Private
	Public
	Protected
	Require
	RequireOnce
	Return
	Static
	Switch
	Throw
	Trait
	Try
	Unset
	Use
	Var
	While
	Yield
	YieldFrom

	//keyword magic constants
	DirectoryConstant
	FileConstant
	LineConstant
	FunctionConstant
	MethodConstant
	NamespaceConstant
	TraitConstant

	//literals
	StringLiteral
	FloatingLiteral
	EncapsulatedAndWhitespace
	Text
	IntegerLiteral

	//Names
	Name
	VariableName

	//Operators and Punctuation
	Equals
	Tilde
	Colon
	Semicolon
	Exclamation
	Dollar
	ForwardSlash
	Percent
	Comma
	AtSymbol
	Backtick
	Question
	DoubleQuote
	SingleQuote
	LessThan
	GreaterThan
	Asterisk
	AmpersandAmpersand
	Ampersand
	AmpersandEquals
	CaretEquals
	LessThanLessThan
	LessThanLessThanEquals
	GreaterThanGreaterThan
	GreaterThanGreaterThanEquals
	BarEquals
	Plus
	PlusEquals
	AsteriskAsterisk
	AsteriskAsteriskEquals
	Arrow
	OpenBrace
	OpenBracket
	OpenParenthesis
	CloseBrace
	CloseBracket
	CloseParenthesis
	QuestionQuestion
	Bar
	BarBar
	Caret
	Dot
	DotEquals
	CurlyOpen
	MinusMinus
	ForwardslashEquals
	DollarCurlyOpen
	FatArrow
	ColonColon
	Ellipsis
	PlusPlus
	EqualsEquals
	GreaterThanEquals
	EqualsEqualsEquals
	ExclamationEquals
	ExclamationEqualsEquals
	LessThanEquals
	Spaceship
	Minus
	MinusEquals
	PercentEquals
	AsteriskEquals
	Backslash
	BooleanCast
	UnsetCast
	StringCast
	ObjectCast
	IntegerCast
	FloatCast
	StartHeredoc
	ArrayCast
	OpenTag
	OpenTagEcho
	CloseTag

	DocumentCommentStart
	DocumentCommentVersion
	DocumentCommentText
	DocumentCommentUnknown
	DocumentCommentStartline
	DocumentCommentEndline
	DocumentCommentTagName

	DocumentCommentTagNameAnchorStart // Anchor for conveniently detecting tags
	AtAuthor
	AtDeprecated
	AtGlobal
	AtLicense
	AtLink
	AtMethod
	AtParam
	AtProperty
	AtPropertyRead
	AtPropertyWrite
	AtReturn
	AtSince
	AtThrows
	AtVar
	DocumentCommentTagNameAnchorEnd

	DocumentCommentEnd

	//Comments whitespace
	Comment
	Whitespace
)

func (*TokenType) MarshalJSON

func (tokenType *TokenType) MarshalJSON() ([]byte, error)

MarshalJSON marshals the token type into JSON

func (TokenType) String

func (i TokenType) String() string