tokenizers

Documentation

Index

• Variables
• type AlphaBoundary
  • func (a *AlphaBoundary) Tokenize(in string) []string
• type RegexpNonWords
  • func (r *RegexpNonWords) Tokenize(in string) []string
• type Tokenizer
• type TokenizerOptions
  • func (o *TokenizerOptions) AddFlags(fs *flag.FlagSet, prefix string)
  • func (o *TokenizerOptions) Validate() error
• type Words
  • func (w *Words) Tokenize(in string) []string

Variables

var AllTokenizers = map[string]Tokenizer{
	"words":         &Words{},
	"nonwords":      &RegexpNonWords{},
	"alphaboundary": &AlphaBoundary{},
}

Types

type AlphaBoundary

type AlphaBoundary struct {
	// contains filtered or unexported fields
}

func (*AlphaBoundary) Tokenize

func (a *AlphaBoundary) Tokenize(in string) []string

type RegexpNonWords

type RegexpNonWords struct{}

func (*RegexpNonWords) Tokenize

func (r *RegexpNonWords) Tokenize(in string) []string

type Tokenizer

type Tokenizer interface {
	Tokenize(string) []string
}

type TokenizerOptions

type TokenizerOptions struct {
	Name *string
}

func (*TokenizerOptions) AddFlags

func (o *TokenizerOptions) AddFlags(fs *flag.FlagSet, prefix string)

func (*TokenizerOptions) Validate

func (o *TokenizerOptions) Validate() error

type Words

type Words struct{}

func (*Words) Tokenize

func (w *Words) Tokenize(in string) []string