tokens

Documentation

Index

• type Pos
  • func (p *Pos) Contains(b *Pos) bool
  • func (p *Pos) Dup() *Pos
  • func (p *Pos) Error(format string, args ...any) error
  • func (a *Pos) Extend(b *Pos) *Pos
  • func (p *Pos) String() string
• type PosError
  • func (p *PosError) Error() string
• type Stream
  • func NewStream(file, code string) *Stream
  • func (s *Stream) CanPeek(off int) bool
  • func (s *Stream) Char() rune
  • func (s *Stream) Eat()
  • func (s *Stream) HasNext() bool
  • func (s *Stream) Peek(off int) rune
  • func (s *Stream) Pos() *Pos
• type Token
  • func (t Token) String() string
• type TokenType
  • func (t TokenType) String() string
• type Tokenizer
  • func NewTokenizer(s *Stream) *Tokenizer
  • func (t *Tokenizer) Eat()
  • func (t *Tokenizer) Filename() string
  • func (t *Tokenizer) HasNext() bool
  • func (t *Tokenizer) Last() *Pos
  • func (t *Tokenizer) Tok() Token
  • func (t *Tokenizer) Tokenize() error

Types

type Pos

type Pos struct {
	File    string
	Line    int
	Char    int
	EndLine int
	EndChar int
}

func (*Pos) Contains

func (p *Pos) Contains(b *Pos) bool

func (*Pos) Dup

func (p *Pos) Dup() *Pos

func (*Pos) Error

func (p *Pos) Error(format string, args ...any) error

func (*Pos) Extend

func (a *Pos) Extend(b *Pos) *Pos

func (*Pos) String

func (p *Pos) String() string

type PosError

type PosError struct {
	Msg string
	Pos *Pos
}

func (*PosError) Error

func (p *PosError) Error() string

type Stream

type Stream struct {
	// contains filtered or unexported fields
}

func NewStream

func NewStream(file, code string) *Stream

func (*Stream) CanPeek

func (s *Stream) CanPeek(off int) bool

func (*Stream) Char

func (s *Stream) Char() rune

func (*Stream) Eat

func (s *Stream) Eat()

func (*Stream) HasNext

func (s *Stream) HasNext() bool

func (*Stream) Peek

func (s *Stream) Peek(off int) rune

func (*Stream) Pos

func (s *Stream) Pos() *Pos

type Token

type Token struct {
	Typ   TokenType
	Value string
	Pos   *Pos
}

func (Token) String

func (t Token) String() string

type TokenType

type TokenType int

const (
	TokenTypeIdent TokenType = iota
	TokenTypeNumber
	TokenTypeString
	TokenTypeByte
	TokenTypeLBrack
	TokenTypeRBrack
)

func (TokenType) String

func (t TokenType) String() string

type Tokenizer

type Tokenizer struct {
	Tokens []Token
	// contains filtered or unexported fields
}

func NewTokenizer

func NewTokenizer(s *Stream) *Tokenizer

func (*Tokenizer) Eat

func (t *Tokenizer) Eat()

func (*Tokenizer) Filename

func (t *Tokenizer) Filename() string

func (*Tokenizer) HasNext

func (t *Tokenizer) HasNext() bool

func (*Tokenizer) Last

func (t *Tokenizer) Last() *Pos

func (*Tokenizer) Tok

func (t *Tokenizer) Tok() Token

func (*Tokenizer) Tokenize

func (t *Tokenizer) Tokenize() error