tokenizer

Documentation

Index

• Variables
• type IsMarkNonSpacingChecker
  • func (c *IsMarkNonSpacingChecker) Contains(r rune) bool
• type SimpleTokenizer
  • func NewSimpleTokenizer() *SimpleTokenizer
  • func (s *SimpleTokenizer) Tokenize(payload []byte) [][]byte
  • func (s *SimpleTokenizer) TokenizeSingle(payload []byte) ([]byte, bool)
• type StopWordsTokenizer
  • func NewStopWordsTokenizer(stopWords []string) *StopWordsTokenizer
  • func (st *StopWordsTokenizer) Tokenize(payload []byte) [][]byte
  • func (st *StopWordsTokenizer) TokenizeSingle(payload []byte) ([]byte, bool)
• type Tokenizer
• type Transformer

Variables

var (
	SpanishStopWords = []string{}/* 443 elements not displayed */

)

var (
	SpanishTokenizer = NewStopWordsTokenizer(SpanishStopWords)
)

Types

type IsMarkNonSpacingChecker

type IsMarkNonSpacingChecker struct{}

func (*IsMarkNonSpacingChecker) Contains

func (c *IsMarkNonSpacingChecker) Contains(r rune) bool

type SimpleTokenizer

type SimpleTokenizer struct {
	Tf Transformer
}

func NewSimpleTokenizer

func NewSimpleTokenizer() *SimpleTokenizer

func (*SimpleTokenizer) Tokenize

func (s *SimpleTokenizer) Tokenize(payload []byte) [][]byte

func (*SimpleTokenizer) TokenizeSingle

func (s *SimpleTokenizer) TokenizeSingle(payload []byte) ([]byte, bool)

type StopWordsTokenizer

type StopWordsTokenizer struct {
	T *SimpleTokenizer
	// contains filtered or unexported fields
}

func NewStopWordsTokenizer

func NewStopWordsTokenizer(stopWords []string) *StopWordsTokenizer

func (*StopWordsTokenizer) Tokenize

func (st *StopWordsTokenizer) Tokenize(payload []byte) [][]byte

func (*StopWordsTokenizer) TokenizeSingle

func (st *StopWordsTokenizer) TokenizeSingle(payload []byte) ([]byte, bool)

type Tokenizer

type Tokenizer interface {
	Tokenize([]byte) [][]byte
	TokenizeSingle([]byte) ([]byte, bool)
}

type Transformer

type Transformer interface {
	Transform(payload []byte) ([]byte, error)
}