text

Documentation

Index

• Constants
• func GetPrefixes(str string) []string
• func GetWords(text string) []string
• func IsRussian(str string) bool
• func MakePrefSuf(str string) [][]string
• func OtherCyr(str string) bool
• func Quoted(text string) string
• func SubStrDistVal(str string, minlen int) map[string]int
• func Truncate(text string, limit int) string
• func WordStream(rdr io.RuneReader, opts ...int) <-chan string

Constants

const (
	WSO_ENDS         int = 0 // ADD dot on ent of statement
	WSO_NO_URLS      int = 1 // SKIP ulrs
	WSO_NO_NUMS      int = 2 // SKIP 123, 12.10, 10:10, 12.08.2018, 10-10 etc
	WSO_HASHTAG      int = 3 // use hashtags
	WSO_NO_XXX_COLON int = 4 // skip xxx: and XXX:
)

const (
	ST_NULL  int = 0
	ST_NUM   int = 1
	ST_ALPHA int = 2

	WS_COMMA string = ","
)

Functions

func GetPrefixes

func GetPrefixes(str string) []string

func GetWords

func GetWords(text string) []string

func IsRussian

func IsRussian(str string) bool

func MakePrefSuf

func MakePrefSuf(str string) [][]string

func OtherCyr

func OtherCyr(str string) bool

func Quoted

func Quoted(text string) string

func SubStrDistVal

func SubStrDistVal(str string, minlen int) map[string]int

func Truncate

func Truncate(text string, limit int) string

func WordStream

func WordStream(rdr io.RuneReader, opts ...int) <-chan string