token

Documentation

Index

• Variables
• type Position
  • func (pos *Position) IsValid() bool
  • func (pos Position) String() string
• type Token
  • func NewToken(typ Type, lit []byte) *Token
  • func (this *Token) Equals(that *Token) bool
  • func (T *Token) IntValue() (int64, error)
  • func (T *Token) SDTVal() string
  • func (this *Token) String() string
  • func (T *Token) UintValue() (uint64, error)
• type TokenMap
  • func NewMap() *TokenMap
  • func NewMapFromFile(file string) (*TokenMap, error)
  • func NewMapFromString(input string) *TokenMap
  • func NewMapFromStrings(input []string) *TokenMap
  • func (this *TokenMap) AddToken(str string)
  • func (this *TokenMap) Equals(that *TokenMap) bool
  • func (this *TokenMap) Len() int
  • func (this *TokenMap) String() string
  • func (this *TokenMap) Strings() []string
  • func (this *TokenMap) TokenString(typ Type) string
  • func (this *TokenMap) Tokens() []*Token
  • func (this *TokenMap) Type(key string) Type
  • func (this *TokenMap) WriteFile(file string) error
• type Type
  • func (T Type) String() string

Variables

var FRONTENDTokens = NewMapFromStrings([]string{
	"id",
	"tokId",
	":",
	";",
	"regDefId",
	"ignoredTokId",
	"|",
	".",
	"char_lit",
	"-",
	"[",
	"]",
	"{",
	"}",
	"(",
	")",
	"prodId",
	"g_sdt_lit",
	"error",
	"empty",
	"string_lit",
})

Types

type Position

type Position struct {
	Offset int // offset, starting at 0
	Line   int // line number, starting at 1
	Column int // column number, starting at 1 (character count)
}

Position describes an arbitrary source position including the file, line, and column location. A Position is valid if the line number is > 0.

func (*Position) IsValid

func (pos *Position) IsValid() bool

IsValid returns true if the position is valid.

func (Position) String

func (pos Position) String() string

String returns a string in one of several forms:

file:line:column    valid position with file name
line:column         valid position without file name
file                invalid position with file name
-                   invalid position without file name

type Token

type Token struct {
	Type Type
	Lit  []byte
}

func NewToken

func NewToken(typ Type, lit []byte) *Token

func (*Token) Equals

func (this *Token) Equals(that *Token) bool

func (*Token) IntValue

func (T *Token) IntValue() (int64, error)

func (*Token) SDTVal

func (T *Token) SDTVal() string

func (*Token) String

func (this *Token) String() string

func (*Token) UintValue

func (T *Token) UintValue() (uint64, error)

type TokenMap

type TokenMap struct {
	// contains filtered or unexported fields
}

func NewMap

func NewMap() *TokenMap

func NewMapFromFile

func NewMapFromFile(file string) (*TokenMap, error)

func NewMapFromString

func NewMapFromString(input string) *TokenMap

func NewMapFromStrings

func NewMapFromStrings(input []string) *TokenMap

func (*TokenMap) AddToken

func (this *TokenMap) AddToken(str string)

func (*TokenMap) Equals

func (this *TokenMap) Equals(that *TokenMap) bool

func (*TokenMap) Len

func (this *TokenMap) Len() int

func (*TokenMap) String

func (this *TokenMap) String() string

func (*TokenMap) Strings

func (this *TokenMap) Strings() []string

func (*TokenMap) TokenString

func (this *TokenMap) TokenString(typ Type) string

func (*TokenMap) Tokens

func (this *TokenMap) Tokens() []*Token

func (*TokenMap) Type

func (this *TokenMap) Type(key string) Type

func (*TokenMap) WriteFile

func (this *TokenMap) WriteFile(file string) error

type Type

type Type int

const (
	ILLEGAL Type = iota - 1
	EOF
)

func (Type) String

func (T Type) String() string