nlp

Documentation

Index

• Constants
• func RegisterTokenizer(lang string, splitter func(string) []string, stemmer func(string) string, ...)
• type Token
• type Tokenizer
  • func FindTokenizer(lang string) (*Tokenizer, error)
  • func (t Tokenizer) StemWord(word string) string
  • func (t Tokenizer) Tokenize(content string, offset int) []*Token
  • func (t Tokenizer) TokenizeToMap(content string, offset int) (tokenMap map[string]*Token, wordCount int)

Constants

const (
	LanguageJapanese = "ja"
	LanguageEnglish  = "en"
)

Functions

func RegisterTokenizer

func RegisterTokenizer(lang string, splitter func(string) []string, stemmer func(string) string, stopWords map[string]bool)

Types

type Token

type Token struct {
	Word      string
	Positions []uint32
}

type Tokenizer

type Tokenizer struct {
	// contains filtered or unexported fields
}

func FindTokenizer

func FindTokenizer(lang string) (*Tokenizer, error)

func (Tokenizer) StemWord

func (t Tokenizer) StemWord(word string) string

func (Tokenizer) Tokenize

func (t Tokenizer) Tokenize(content string, offset int) []*Token

func (Tokenizer) TokenizeToMap

func (t Tokenizer) TokenizeToMap(content string, offset int) (tokenMap map[string]*Token, wordCount int)