character

Documentation

Index

• type CharacterTokenizer
  • func NewCharacterTokenizer(f IsTokenRune) *CharacterTokenizer
  • func (c *CharacterTokenizer) Tokenize(input []byte) analysis.TokenStream
• type IsTokenRune

Types

type CharacterTokenizer

type CharacterTokenizer struct {
	// contains filtered or unexported fields
}

func NewCharacterTokenizer

func NewCharacterTokenizer(f IsTokenRune) *CharacterTokenizer

func (*CharacterTokenizer) Tokenize

func (c *CharacterTokenizer) Tokenize(input []byte) analysis.TokenStream

type IsTokenRune

type IsTokenRune func(r rune) bool