token

Documentation

Index

• Variables
• type Item
  • func Keyword(t Token) Item
  • func NewItem(t Token, v string) Item
  • func (i Item) Position() Position
  • func (i Item) String() string
• type ItemList
  • func NewList(t ...Item) *ItemList
  • func (s *ItemList) Abort()
  • func (s *ItemList) Next() Item
  • func (s *ItemList) Peek() Item
  • func (s *ItemList) Previous() Item
  • func (s *ItemList) Push(i ...Item)
  • func (s *ItemList) PushKeyword(t Token)
  • func (s *ItemList) PushStream(i Stream)
  • func (s *ItemList) Seek(position int)
• type Position
• type Stream
  • func Subset(s Stream, t Type) Stream
• type Token
  • func (i Token) String() string
  • func (t Token) Type() Type
• type Type
  • func (t Type) Is(ty Type) bool

Variables

var OperatorMarks = map[rune]struct{}{}

OperatorMarks represents a list of operators

var TokenList []string

TokenList represents a list of tokens

var TokenMap = map[string]Token{}/* 133 elements not displayed */

TokenMap maps source code string tokens to types when strings can be represented directly. Not all types will be represented here.

Types

type Item

type Item struct {
	Typ        Token
	Begin, End Position
	Val        string
}

Item represents a lexed item.

func Keyword

func Keyword(t Token) Item

Keyword returns a Keyword

func NewItem

func NewItem(t Token, v string) Item

NewItem returns a new Item

func (Item) Position

func (i Item) Position() Position

func (Item) String

func (i Item) String() string

String renders a string representation of the item.

type ItemList

type ItemList struct {
	// Items contains all the items in the list.
	Items []Item

	// Position is the current position the set is at in the token slice.
	Position int
}

ItemList represents an ordered set of tokens.

func NewList

func NewList(t ...Item) *ItemList

NewList initializes a new ItemList

func (*ItemList) Abort

func (s *ItemList) Abort()

func (*ItemList) Next

func (s *ItemList) Next() Item

Next consumes and returns the next item in the list.

func (*ItemList) Peek

func (s *ItemList) Peek() Item

func (*ItemList) Previous

func (s *ItemList) Previous() Item

func (*ItemList) Push

func (s *ItemList) Push(i ...Item)

func (*ItemList) PushKeyword

func (s *ItemList) PushKeyword(t Token)

func (*ItemList) PushStream

func (s *ItemList) PushStream(i Stream)

func (*ItemList) Seek

func (s *ItemList) Seek(position int)

type Position

type Position struct {
	Line, Column int // The position relative to other characters in the file
	Position     int // The position in bytes in the file
	File         string
}

Position is a position

type Stream

type Stream interface {
	// Next consumes and returns the next item in the stream. If there is no next
	// item, the zero value is returned.
	Next() Item

	// Previous consumes and returns the previous item in the stream. i.e.
	//
	// 	a := s.Next()
	//	b := s.Previous()
	//	c := s.Next()
	//	a == b == c // true
	//
	// If there is no previous item, the zero value is returned.
	Previous() Item

	Abort()
}

Stream is an ordered set of tokens

func Subset

func Subset(s Stream, t Type) Stream

Subset returns a stream that emits only tokens from s that are of Type t. If s is already a subset, it is re-expanded before the new subset is created.

type Token

type Token int

Token is a token

const (
	EOF Token = iota
	HTML
	PHPBegin
	PHPEnd
	PHPToken
	Error
	Space
	Function
	Static
	Self
	Parent
	Final
	FunctionName
	TypeHint
	VariableOperator
	BlockBegin
	BlockEnd
	Global

	Namespace
	Use

	CommentLine
	CommentBlock

	IgnoreErrorOperator

	Return
	Comma
	StatementEnd
	Echo
	Print

	If
	Else
	ElseIf
	For
	Foreach
	EndIf
	EndFor
	EndForeach
	EndWhile
	EndSwitch
	AsOperator
	While
	Continue
	Break
	Do
	OpenParen
	CloseParen
	Switch
	Case
	Default

	Try
	Catch
	Finally
	Throw

	Class
	Abstract
	Private
	Public
	Protected
	Interface
	Implements
	Extends
	NewOperator
	Const

	Null
	StringLiteral
	NumberLiteral
	BooleanLiteral

	ShellCommand

	Identifier

	AssignmentOperator
	NegationOperator
	AdditionOperator
	SubtractionOperator
	MultOperator
	ConcatenationOperator
	UnaryOperator
	ComparisonOperator
	InstanceofOperator

	AndOperator
	OrOperator
	WrittenAndOperator
	WrittenXorOperator
	WrittenOrOperator

	ObjectOperator
	ScopeResolutionOperator

	CastOperator

	Var
	Array
	ArrayKeyOperator
	ArrayLookupOperatorLeft
	ArrayLookupOperatorRight
	List
	BitwiseShiftOperator
	StrongEqualityOperator
	StrongNotEqualityOperator
	EqualityOperator
	NotEqualityOperator
	AmpersandOperator
	BitwiseXorOperator
	BitwiseOrOperator
	BitwiseNotOperator
	TernaryOperator1
	TernaryOperator2

	Declare

	Include
	Exit
)

func (Token) String

func (i Token) String() string

func (Token) Type

func (t Token) Type() Type

Type returns the type of t.

type Type

type Type int

Type is a bitmask representing the type of a token

const (
	InvalidType Type = 1 << iota

	KeywordType    // keyword, e.g. "static", "function"
	LiteralType    // literal, e.g. 234, "a string", false
	MarkerType     // marker for code blocks and groupings, e.g. {, (
	OperatorType   // operator, e.g. +, ===, $
	IdentifierType // identifier, e.g. StdClass

	CommentType
	WhitespaceType

	Significant = KeywordType | LiteralType | MarkerType | OperatorType | IdentifierType
)

func (Type) Is

func (t Type) Is(ty Type) bool

Is returns true if ty is equal to or contained by t.