nlp_summarize

Documentation

Index

• Constants
• func CalculateSummarySentences(text string, value float32) int
• func DefaultWordTokenizer(sentence string) []string
• func Distance(str1 string, str2 string) float64
• func HighSummarySentences(text string) int
• func Intersection(src, dst []int) []int
• func LowSummarySentences(text string) int
• func MediumSummarySentences(text string) int
• func ReverseEdge(num []*Edge)
• func ReverseRank(num []*Rank)
• func SanitizeWord(word string) string
• func SymmetricDifference(src, dst []int) []int
• func TokenizeSentences(text string) []string
• func UniqSentences(sentences [][]string, sentenceDistanceThreshold float64)
• type ByScore
  • func (b ByScore) Len() int
  • func (b ByScore) Less(i, j int) bool
  • func (b ByScore) Swap(i, j int)
• type ByWeight
  • func (b ByWeight) Len() int
  • func (b ByWeight) Less(i, j int) bool
  • func (b ByWeight) Swap(i, j int)
• type Edge
• type Node
• type Rank
• type Summarizer
  • func NewSummarizer() *Summarizer
  • func (instance *Summarizer) Set(m int, d, t, th, sth float64, alg, w string)
  • func (instance *Summarizer) SetCustomAlgorithm(f func(e []*Edge) []int)
  • func (instance *Summarizer) SetCustomWeighing(f func(src, dst []int) float64)
  • func (instance *Summarizer) SetDictionary(dict map[string]int)
  • func (instance *Summarizer) SetWordTokenizer(f func(string) []string)
  • func (instance *Summarizer) String() string
  • func (instance *Summarizer) Summarize(text string, num int) ([]string, error)

Constants

const (
	AlgPageRank   = "pagerank"
	AlgCentrality = "centrality"
	AlgCustom     = "custom"

	EdgJaccard = "jaccard"
	EdgHamming = "hamming"
	EdgCustom  = "custom"

	DEFAULT_ALGORITHM                    = AlgPageRank
	DEFAULT_WEIGHING                     = EdgHamming
	DEFAULT_DAMPING                      = 0.85
	DEFAULT_TOLERANCE                    = 0.0001
	DEFAULT_THRESHOLD                    = 0.001
	DEFAULT_MAX_CHARACTERS               = 0
	DEFAULT_SENTENCES_DISTANCE_THRESHOLD = 0.95
)

Functions

func CalculateSummarySentences

func CalculateSummarySentences(text string, value float32) int

func DefaultWordTokenizer

func DefaultWordTokenizer(sentence string) []string

func Distance

func Distance(str1 string, str2 string) float64

Distance Whole code adapted to Go from: https://github.com/NaturalNode/natural/blob/master/lib/natural/distance/jaro-winkler_distance.js

func HighSummarySentences

func HighSummarySentences(text string) int

HighSummarySentences Calculate summary sentences Response is about 15% of original text

func Intersection

func Intersection(src, dst []int) []int

func LowSummarySentences

func LowSummarySentences(text string) int

func MediumSummarySentences

func MediumSummarySentences(text string) int

func ReverseEdge

func ReverseEdge(num []*Edge)

func ReverseRank

func ReverseRank(num []*Rank)

func SanitizeWord

func SanitizeWord(word string) string

func SymmetricDifference

func SymmetricDifference(src, dst []int) []int

func TokenizeSentences

func TokenizeSentences(text string) []string

func UniqSentences

func UniqSentences(sentences [][]string, sentenceDistanceThreshold float64)

Types

type ByScore

type ByScore []*Rank

func (ByScore) Len

func (b ByScore) Len() int

func (ByScore) Less

func (b ByScore) Less(i, j int) bool

func (ByScore) Swap

func (b ByScore) Swap(i, j int)

type ByWeight

type ByWeight []*Edge

func (ByWeight) Len

func (b ByWeight) Len() int

func (ByWeight) Less

func (b ByWeight) Less(i, j int) bool

func (ByWeight) Swap

func (b ByWeight) Swap(i, j int)

type Edge

type Edge struct {
	// contains filtered or unexported fields
}

type Node

type Node struct {
	// contains filtered or unexported fields
}

type Rank

type Rank struct {
	// contains filtered or unexported fields
}

type Summarizer

type Summarizer struct {
	BagOfWordsPerSentence [][]string
	OriginalSentences     []string
	Dict                  map[string]int
	Nodes                 []*Node
	Edges                 []*Edge
	Ranks                 []int

	MaxCharacters              int
	Algorithm                  string // "centrality" or "pagerank" or "custom"
	Weighing                   string // "hamming" or "jaccard" or "custom"
	Damping                    float64
	Tolerance                  float64
	Threshold                  float64
	SentencesDistanceThreshold float64
	// contains filtered or unexported fields
}

func NewSummarizer

func NewSummarizer() *Summarizer

NewSummarizer creates a new summarizer

func (*Summarizer) Set

func (instance *Summarizer) Set(m int, d, t, th, sth float64, alg, w string)

Set max characters, damping, tolerance, threshold, sentences distance threshold, algorithm, and weighing

func (*Summarizer) SetCustomAlgorithm

func (instance *Summarizer) SetCustomAlgorithm(f func(e []*Edge) []int)

func (*Summarizer) SetCustomWeighing

func (instance *Summarizer) SetCustomWeighing(f func(src, dst []int) float64)

func (*Summarizer) SetDictionary

func (instance *Summarizer) SetDictionary(dict map[string]int)

SetDictionary Useful if you already have your own dictionary (example: from your database) Dictionary is a map[string]int where the key is the word and int is the position in vector, starting from 1

func (*Summarizer) SetWordTokenizer

func (instance *Summarizer) SetWordTokenizer(f func(string) []string)

func (*Summarizer) String

func (instance *Summarizer) String() string

func (*Summarizer) Summarize

func (instance *Summarizer) Summarize(text string, num int) ([]string, error)

Summarize the text to num sentences