token

Documentation

Index

• Variables
• type DirectivePredicator
  • func NewDirectivePredicator(nameSet ext.StringSet) *DirectivePredicator
  • func (m *DirectivePredicator) Name() string
  • func (m *DirectivePredicator) Predicate(t *Token) bool
• type Predicator
  • func Directives(names ...string) Predicator
  • func Every(predicators ...Predicator) Predicator
  • func Not(p Predicator) Predicator
  • func PortabilityDirectives(names ...string) Predicator
  • func PredicatorBy(name string, fn func(string) bool) Predicator
  • func Some(predicators ...Predicator) Predicator
  • func Symbol(rs ...rune) Predicator
  • func TokenType(typ Type) Predicator
  • func UpperCase(v string) Predicator
  • func Value(v string) Predicator
• type PredicatorImpl
  • func (p *PredicatorImpl) Name() string
  • func (p *PredicatorImpl) Predicate(token *Token) bool
• type Predicators
  • func (s Predicators) Every(t *Token) bool
  • func (s Predicators) Names() []string
  • func (s Predicators) Some(t *Token) bool
• type Token
  • func NewToken(typ Type, text *[]rune, start, end *runes.Position) *Token
  • func ProcessComment(c *runes.Cursor) *Token
  • func ProcessDoubleSpecialSymbol(c *runes.Cursor) *Token
  • func ProcessEof(c *runes.Cursor) *Token
  • func ProcessNumeral(c *runes.Cursor) *Token
  • func ProcessSingleSpecialSymbol(c *runes.Cursor) *Token
  • func ProcessSpace(c *runes.Cursor) *Token
  • func ProcessString(c *runes.Cursor) *Token
  • func ProcessWord(c *runes.Cursor) *Token
  • func (t *Token) Clone() *Token
  • func (t *Token) Is(pred Predicator) bool
  • func (t *Token) Len() int
  • func (t *Token) Raw() []rune
  • func (t *Token) RawString() string
  • func (t *Token) String() string
  • func (t *Token) Value() string
  • func (t *Token) ValueAbbr(n int) string
• type TokeninzerFlag
• type Tokenizer
  • func NewTokenizer(text *[]rune, flags TokeninzerFlag) *Tokenizer
  • func (t *Tokenizer) Clone() *Tokenizer
  • func (t *Tokenizer) GetNext() *Token
• type Type
  • func (typ Type) HasKeyword(kw string) Predicator
  • func (typ Type) HasText(s string) Predicator
  • func (typ Type) Name() string
  • func (typ Type) Predicate(t *Token) bool
  • func (t Type) String() string

Variables

var Directive = NewDirectivePredicator(directives)

var PortabilityDirective = NewDirectivePredicator(portabilityDirectives)

var SingleSpecialSymbols = map[rune]bool{
	'#':  true,
	'$':  true,
	'&':  true,
	'\'': true,
	'(':  true,
	')':  true,
	'*':  true,
	'+':  true,
	',':  true,
	'-':  true,
	'.':  true,
	'/':  true,
	':':  true,
	';':  true,
	'<':  true,
	'=':  true,
	'>':  true,
	'@':  true,
	'[':  true,
	']':  true,
	'^':  true,
	'{':  true,
	'}':  true,
}

var TwoRunesSpecialSymbols = map[[2]rune]bool{

	{'(', '.'}: true,

	{'.', ')'}: true,
	{'.', '.'}: true,

	{':', '='}: true,
	{'<', '='}: true,
	{'>', '='}: true,
	{'<', '>'}: true,
}

var TypeNames = map[Type]string{
	EOF:                 "EOF",
	Space:               "space",
	Comment:             "comment",
	SpecialSymbol:       "special symbol",
	Identifier:          "identifier",
	QualifiedIdentifier: "qualified identifier",
	ReservedWord:        "reserved word",
	NumeralInt:          "int",
	NumeralReal:         "real",
	Label:               "label",
	CharacterString:     "character string",
}

Types

type DirectivePredicator

type DirectivePredicator struct {
	// contains filtered or unexported fields
}

func NewDirectivePredicator

func NewDirectivePredicator(nameSet ext.StringSet) *DirectivePredicator

func (*DirectivePredicator) Name

func (m *DirectivePredicator) Name() string

func (*DirectivePredicator) Predicate

func (m *DirectivePredicator) Predicate(t *Token) bool

type Predicator

type Predicator interface {
	Name() string
	Predicate(*Token) bool
}

func Directives

func Directives(names ...string) Predicator

func Every

func Every(predicators ...Predicator) Predicator

func Not

func Not(p Predicator) Predicator

func PortabilityDirectives

func PortabilityDirectives(names ...string) Predicator

func PredicatorBy

func PredicatorBy(name string, fn func(string) bool) Predicator

func Some

func Some(predicators ...Predicator) Predicator

func Symbol

func Symbol(rs ...rune) Predicator

func TokenType

func TokenType(typ Type) Predicator

func UpperCase

func UpperCase(v string) Predicator

func Value

func Value(v string) Predicator

type PredicatorImpl

type PredicatorImpl struct {
	// contains filtered or unexported fields
}

func (*PredicatorImpl) Name

func (p *PredicatorImpl) Name() string

func (*PredicatorImpl) Predicate

func (p *PredicatorImpl) Predicate(token *Token) bool

type Predicators

type Predicators []Predicator

func (Predicators) Every

func (s Predicators) Every(t *Token) bool

func (Predicators) Names

func (s Predicators) Names() []string

func (Predicators) Some

func (s Predicators) Some(t *Token) bool

type Token

type Token struct {
	Type Type

	Start *runes.Position
	End   *runes.Position
	// contains filtered or unexported fields
}

func NewToken

func NewToken(typ Type, text *[]rune, start, end *runes.Position) *Token

func ProcessComment

func ProcessComment(c *runes.Cursor) *Token

func ProcessDoubleSpecialSymbol

func ProcessDoubleSpecialSymbol(c *runes.Cursor) *Token

func ProcessEof

func ProcessEof(c *runes.Cursor) *Token

func ProcessNumeral

func ProcessNumeral(c *runes.Cursor) *Token

func ProcessSingleSpecialSymbol

func ProcessSingleSpecialSymbol(c *runes.Cursor) *Token

func ProcessSpace

func ProcessSpace(c *runes.Cursor) *Token

func ProcessString

func ProcessString(c *runes.Cursor) *Token

func ProcessWord

func ProcessWord(c *runes.Cursor) *Token

func (*Token) Clone

func (t *Token) Clone() *Token

func (*Token) Is

func (t *Token) Is(pred Predicator) bool

func (*Token) Len

func (t *Token) Len() int

func (*Token) Raw

func (t *Token) Raw() []rune

func (*Token) RawString

func (t *Token) RawString() string

func (*Token) String

func (t *Token) String() string

func (*Token) Value

func (t *Token) Value() string

func (*Token) ValueAbbr

func (t *Token) ValueAbbr(n int) string

type TokeninzerFlag

type TokeninzerFlag uint

const (
	LoadSpace TokeninzerFlag = 1 << iota
	LoadComment
)

type Tokenizer

type Tokenizer struct {
	*runes.Cursor
	// contains filtered or unexported fields
}

func NewTokenizer

func NewTokenizer(text *[]rune, flags TokeninzerFlag) *Tokenizer

func (*Tokenizer) Clone

func (t *Tokenizer) Clone() *Tokenizer

func (*Tokenizer) GetNext

func (t *Tokenizer) GetNext() *Token

type Type

type Type uint8

const (
	EOF Type = iota // 0
	Space
	Comment
	SpecialSymbol
	Identifier
	QualifiedIdentifier
	ReservedWord
	NumeralInt
	NumeralReal
	Label
	CharacterString
)

func (Type) HasKeyword

func (typ Type) HasKeyword(kw string) Predicator

kw must be upper case

func (Type) HasText

func (typ Type) HasText(s string) Predicator

func (Type) Name

func (typ Type) Name() string

func (Type) Predicate

func (typ Type) Predicate(t *Token) bool

func (Type) String

func (t Type) String() string