token

Documentation

Index

• Variables
• func EndsWith(ts []Token, pat []Token) bool
• func HasPattern(ts []Token, pat []TokenType) bool
• func Literals(ts []Token) string
• func Split(ts []Token, tt TokenType) [][]Token
• func SplitOnFirst(ts []Token, tt TokenType) ([]Token, []Token)
• func StartsWith(ts []Token, pat []Token) bool
• type Lexer
  • func NewLexer(in io.Reader) *Lexer
  • func (lex *Lexer) Error() error
  • func (lex *Lexer) Out() chan Token
• type Token
  • func (t Token) Equal(wanted ...Token) bool
  • func (t Token) Len() int
• type TokenReader
  • func NewTokenReader(in chan Token) *TokenReader
  • func (tr *TokenReader) Discard()
  • func (tr *TokenReader) Read(n int) ([]Token, bool)
  • func (tr *TokenReader) Unread(n int) bool
• type TokenStream
  • func NewTokenStream(toks []Token) *TokenStream
  • func (s *TokenStream) Out() chan Token
• type TokenType
  • func FindTokenType(ch rune) TokenType

Variables

var (
	TOKEN_EOF            = Token{Type: TYPE_EOF}
	TOKEN_EOS            = Token{Type: TYPE_EOS}
	TOKEN_NEWLINE        = Token{Type: TYPE_NEWLINE, Literal: "\n"}
	TOKEN_DOUBLE_NEWLINE = Token{Type: TYPE_NEWLINE, Literal: "\n\n"}
	TOKEN_ASTERISK       = Token{Type: TYPE_ASTERISK, Literal: "*"}
)

var TokenTypeMap = map[TokenType][]rune{
	TYPE_EOF:          {eof},
	TYPE_WHITESPACE:   {' ', '\t'},
	TYPE_NEWLINE:      {'\n'},
	TYPE_EQUALSIGN:    {'='},
	TYPE_BACKTICK:     {'`'},
	TYPE_DASH:         {'-'},
	TYPE_COLON:        {':'},
	TYPE_ASTERISK:     {'*'},
	TYPE_UNDERSCORE:   {'_'},
	TYPE_BRACKETOPEN:  {'['},
	TYPE_BRACKETCLOSE: {']'},
}

Functions

func EndsWith

func EndsWith(ts []Token, pat []Token) bool

func HasPattern

func HasPattern(ts []Token, pat []TokenType) bool

func Literals

func Literals(ts []Token) string

func Split

func Split(ts []Token, tt TokenType) [][]Token

func SplitOnFirst

func SplitOnFirst(ts []Token, tt TokenType) ([]Token, []Token)

func StartsWith

func StartsWith(ts []Token, pat []Token) bool

Types

type Lexer

type Lexer struct {
	// contains filtered or unexported fields
}

func NewLexer

func NewLexer(in io.Reader) *Lexer

func (*Lexer) Error

func (lex *Lexer) Error() error

func (*Lexer) Out

func (lex *Lexer) Out() chan Token

type Token

type Token struct {
	Type    TokenType
	Literal string
}

func (Token) Equal

func (t Token) Equal(wanted ...Token) bool

func (Token) Len

func (t Token) Len() int

type TokenReader

type TokenReader struct {
	// contains filtered or unexported fields
}

func NewTokenReader

func NewTokenReader(in chan Token) *TokenReader

func (*TokenReader) Discard

func (tr *TokenReader) Discard()

func (*TokenReader) Read

func (tr *TokenReader) Read(n int) ([]Token, bool)

func (*TokenReader) Unread

func (tr *TokenReader) Unread(n int) bool

type TokenStream

type TokenStream struct {
	// contains filtered or unexported fields
}

func NewTokenStream

func NewTokenStream(toks []Token) *TokenStream

func (*TokenStream) Out

func (s *TokenStream) Out() chan Token

type TokenType

type TokenType int

const (
	TYPE_EOF TokenType = iota
	TYPE_EOS
	TYPE_WHITESPACE
	TYPE_NEWLINE

	TYPE_EQUALSIGN
	TYPE_ASTERISK
	TYPE_UNDERSCORE
	TYPE_BACKTICK
	TYPE_DASH
	TYPE_COLON
	TYPE_BRACKETOPEN
	TYPE_BRACKETCLOSE

	TYPE_WORD
)

func FindTokenType

func FindTokenType(ch rune) TokenType