Documentation
¶
Index ¶
Constants ¶
This section is empty.
Variables ¶
View Source
var (
SpanishStopWords = StopWords{}/* 443 elements not displayed */
)
Functions ¶
This section is empty.
Types ¶
type CleanTokenizer ¶
type CleanTokenizer struct {
// contains filtered or unexported fields
}
func NewCleanTokenizer ¶
func NewCleanTokenizer(fns ...cleanFunc) CleanTokenizer
func NewKeepAlphanumericTokenizer ¶
func NewKeepAlphanumericTokenizer() CleanTokenizer
func (*CleanTokenizer) Tokenize ¶
func (c *CleanTokenizer) Tokenize(text string) []string
type LowerCaseFilter ¶
type LowerCaseFilter struct{}
func NewLowerCaseTokenizer ¶
func NewLowerCaseTokenizer() LowerCaseFilter
func (LowerCaseFilter) Filter ¶
func (l LowerCaseFilter) Filter(tokens []string) []string
type SpanishStemmerFilter ¶
type SpanishStemmerFilter struct {
// contains filtered or unexported fields
}
func NewSpanishStemmer ¶
func NewSpanishStemmer(removeStopWords bool) SpanishStemmerFilter
func (SpanishStemmerFilter) Filter ¶
func (s SpanishStemmerFilter) Filter(tokens []string) []string
type StopWordsFilter ¶
type StopWordsFilter struct {
// contains filtered or unexported fields
}
func NewStopWordsFilter ¶
func NewStopWordsFilter(sw StopWords) StopWordsFilter
func (StopWordsFilter) Filter ¶
func (s StopWordsFilter) Filter(tokens []string) []string
type TokenizationPipeline ¶
type TokenizationPipeline struct {
// contains filtered or unexported fields
}
func NewTokenizationPipeline ¶
func NewTokenizationPipeline(t Tokenizer, f ...Filter) TokenizationPipeline
func (*TokenizationPipeline) Tokenize ¶
func (p *TokenizationPipeline) Tokenize(text string) []string
Source Files
Click to show internal directories.
Click to hide internal directories.