lexer

Documentation

Index

• func Lex(src *bufio.Reader, tokens chan<- Token)
• type Token
• type TokenKind

Functions

func Lex

func Lex(src *bufio.Reader, tokens chan<- Token)

Lex converts a source into a stream of tokens.

Types

type Token

type Token struct {
	Kind  TokenKind
	Value string
	Err   error

	Line, Col int
}

A Token is a single lexical element.

type TokenKind

type TokenKind int

TokenKind tells what kind of value is in the Token

const (
	// BadToken is bad.. too bad..
	BadToken TokenKind = iota
	// PunctuatorToken has special characters
	PunctuatorToken // !	$	(	)	...	:	=	@	[	]	{	|	}
	// NameToken has names
	NameToken // /[_A-Za-z][_0-9A-Za-z]*/
	// IntValueToken has iteger numbers
	IntValueToken // NegativeSign(opt) | NonZeroDigit | Digit (list, opt)
	// FloatValueToken has float numbers
	FloatValueToken // Sign (opt) | IntegerPart | FractionalPart (ExponentPart)
	// StringValueToken has string values
	StringValueToken // "something which is a string"
	// UnicodeBOMToken is just the \ufeff
	UnicodeBOMToken // \ufeff
	// WhitespaceToken is \t and 'space'
	WhitespaceToken // \t and 'space'
	// LineTerminatorToken is \n and \r
	LineTerminatorToken // \n
	// CommentToken is # something like this
	CommentToken // # Just a comment....
	// CommaToken is just a ','
	CommaToken // ,
)