util

Documentation

Index

• Variables
• func Normalize(word string, reg *stopwords.Register) (string, bool)
• func Sanitize(strs [][]byte, reg *stopwords.Register) []string
• func SortTagItems(items []*model.Tag)
• func SplitToSentences(text []byte) [][]byte
• func SplitToTokens(text []byte, cfg *config.Config) []string
• func TFIDF(t *model.Tag) float64
• func UpdateControlStr(candidate, control string) string

Variables

var (
	Domains = stopwords.Setup(stopwords.WithDomains(true))
)

Functions

func Normalize

func Normalize(word string, reg *stopwords.Register) (string, bool)

Normalize sanitizes word and tells whether it is allowed token or not.

func Sanitize

func Sanitize(strs [][]byte, reg *stopwords.Register) []string

Sanitize ...

func SortTagItems

func SortTagItems(items []*model.Tag)

SortTagItems sorts items by score in descending order, if scores are equal it sorts by count if counts are equal, it sorts string values alphabetically.

func SplitToSentences

func SplitToSentences(text []byte) [][]byte

SplitToSentences splits given text into slice of sentences.

func SplitToTokens

func SplitToTokens(text []byte, cfg *config.Config) []string

func TFIDF

func TFIDF(t *model.Tag) float64

TFIDF applies TF-IDF to given Tag

func UpdateControlStr

func UpdateControlStr(candidate, control string) string