nlp

Documentation

Index

• func NewNLPTokenizer(credentialsFilename string, lang Lang) tokenize.Tokenizer
• type Lang
• type NLPPoSDetermer
  • func NewNLPPoSDetermer(poS tokenize.PoS) NLPPoSDetermer
  • func (dps NLPPoSDetermer) DetermPoS(textTokens []tokenize.Token, entityTokens [][]tokenize.Token) []tokenize.Token
• type NLPTokenizer
  • func (nlp NLPTokenizer) Tokenize(ctx context.Context, text string) ([]tokenize.Token, error)

Functions

func NewNLPTokenizer

func NewNLPTokenizer(credentialsFilename string, lang Lang) tokenize.Tokenizer

NewNLPTokenizer returns a new NLP tokenizer instance

Types

type Lang

type Lang string

Lang defines the language used to examine the text. Both ISO and BCP-47 language codes are accepted

var AutoLang Lang = "auto"

AutoLang tries to automatically recognize the language

type NLPPoSDetermer

type NLPPoSDetermer struct {
	// contains filtered or unexported fields
}

NLPPoSDetermer represents the default part of speech determinator

func NewNLPPoSDetermer

func NewNLPPoSDetermer(poS tokenize.PoS) NLPPoSDetermer

NewNLPPoSDetermer returns a new default part of speech determinator

func (NLPPoSDetermer) DetermPoS

func (dps NLPPoSDetermer) DetermPoS(textTokens []tokenize.Token, entityTokens [][]tokenize.Token) []tokenize.Token

DetermPoS deterimantes if a part of speech tag should be kept. It always appends entities

type NLPTokenizer

type NLPTokenizer struct {
	// contains filtered or unexported fields
}

NLPTokenizer tokenizes a text using Google NLP

func (NLPTokenizer) Tokenize

func (nlp NLPTokenizer) Tokenize(ctx context.Context, text string) ([]tokenize.Token, error)

Tokenize tokenizes a text