dictionary

Documentation

Index

• func LoadDictKeys(appConfig config.AppConfig) (*map[string]bool, error)
• func Ngrams(chars []string, minLen int) []string
• func ValidateDict(wdict map[string]*dicttypes.Word, validator Validator) error
• type Dictionary
  • func LoadDictFile(appConfig config.AppConfig) (*Dictionary, error)
  • func LoadDictURL(appConfig config.AppConfig, url string) (*Dictionary, error)
  • func NewDictionary(wdict map[string]*dicttypes.Word) *Dictionary
• type FsClient
• type HeadwordSubstrings
• type NotesExtractor
  • func NewNotesExtractor(patternList string) (*NotesExtractor, error)
  • func (n NotesExtractor) Extract(notes string) []string
• type NotesProcessor
  • func NewNotesProcessor(patternList, replaceList string) NotesProcessor
  • func (p NotesProcessor) Process(w dicttypes.Word) dicttypes.Word
• type Results
• type ReverseIndex
  • func NewReverseIndex(dict *Dictionary, nExtractor *NotesExtractor) ReverseIndex
• type SubstringIndex
  • func NewSubstringIndexFS(client FsClient, corpus string, generation int, dict *Dictionary) (SubstringIndex, error)
  • func NewSubstringIndexMem(ctx context.Context) (SubstringIndex, error)
• type SubstringIndexMem
  • func (searcher SubstringIndexMem) LookupSubstr(ctx context.Context, query, topic_en, subtopic_en string) (*Results, error)
• type Validator
  • func NewValidator(posReader io.Reader, domainReader io.Reader) (Validator, error)

Functions

func LoadDictKeys

func LoadDictKeys(appConfig config.AppConfig) (*map[string]bool, error)

LoadDictKeys loads the keys only from static files

func Ngrams

func Ngrams(chars []string, minLen int) []string

Ngrams finds the set of all substrings in the array longer than minLen characters

func ValidateDict

func ValidateDict(wdict map[string]*dicttypes.Word, validator Validator) error

ValidateDict check the Chinese-English for errors

Types

type Dictionary

type Dictionary struct {
	// Forward dictionary, lookup by Chinese word
	Wdict       map[string]*dicttypes.Word
	HeadwordIds map[int]*dicttypes.Word
}

Dictionary is a struct to hold word dictionary indexes

func LoadDictFile

func LoadDictFile(appConfig config.AppConfig) (*Dictionary, error)

LoadDictFile loads all words from static files

func LoadDictURL

func LoadDictURL(appConfig config.AppConfig, url string) (*Dictionary, error)

Loads all words from a URL

func NewDictionary

func NewDictionary(wdict map[string]*dicttypes.Word) *Dictionary

type FsClient

type FsClient interface {
	Collection(path string) *firestore.CollectionRef
}

FsClient defines Firestore interfaces needed

type HeadwordSubstrings

type HeadwordSubstrings struct {
	HeadwordId  int64    `firestore:"headword_id"`
	Simplified  string   `firestore:"simplified"`
	Traditional string   `firestore:"traditional"`
	Pinyin      string   `firestore:"pinyin"`
	Substrings  []string `firestore:"substrings"`
}

HeadwordSubstrings holds substrings of a headword

type NotesExtractor

type NotesExtractor struct {
	// contains filtered or unexported fields
}

NotesExtractor is an interface for extracting multilingual equivalents using regular expressions in the notes.

func NewNotesExtractor

func NewNotesExtractor(patternList string) (*NotesExtractor, error)

NewNotesExtractor creates a new NotesExtractor.

func (NotesExtractor) Extract

func (n NotesExtractor) Extract(notes string) []string

Extract extracts multilingual equivalents from the given note

type NotesProcessor

type NotesProcessor struct {
	// contains filtered or unexported fields
}

NotesProcessor processes notes with a regular expression

func NewNotesProcessor

func NewNotesProcessor(patternList, replaceList string) NotesProcessor

newNotesProcessor creates a new notesProcessor Param

patternList a list of patterns to match regular expressions, quoted and delimited by commas
replaceList a list of replacement regular expressions, same cardinality

func (NotesProcessor) Process

func (p NotesProcessor) Process(w dicttypes.Word) dicttypes.Word

Process checks all senses in the word and replaces note using the regex

type Results

type Results struct {
	Words []dicttypes.Word
}

Encapsulates term lookup recults

type ReverseIndex

type ReverseIndex interface {
	// Find searches from English, pinyin, or multilingual equivalents contained in notes to Chinese
	Find(ctx context.Context, query string) ([]dicttypes.WordSense, error)
}

ReverseIndex searches the dictionary by reverse lookup, eg to Chinese

func NewReverseIndex

func NewReverseIndex(dict *Dictionary, nExtractor *NotesExtractor) ReverseIndex

type SubstringIndex

type SubstringIndex interface {
	LookupSubstr(ctx context.Context, query, topic_en, subtopic_en string) (*Results, error)
}

func NewSubstringIndexFS

func NewSubstringIndexFS(client FsClient, corpus string, generation int, dict *Dictionary) (SubstringIndex, error)

NewSubstringIndexDB initialize an implementation of SubstringIndex using the index saved in Firestore

func NewSubstringIndexMem

func NewSubstringIndexMem(ctx context.Context) (SubstringIndex, error)

NewSubstringIndexMem initialize a SubstringIndexMem

type SubstringIndexMem

type SubstringIndexMem struct {
}

SubstringIndexMem looks up substrings from a map loaded from a file.

func (SubstringIndexMem) LookupSubstr

func (searcher SubstringIndexMem) LookupSubstr(ctx context.Context, query, topic_en, subtopic_en string) (*Results, error)

Lookup a term based on a substring and a topic

type Validator

type Validator interface {
	Validate(pos, domain string) error
}

Performs validation of dictionary entries. Use NewValidator to create a Validator.

func NewValidator

func NewValidator(posReader io.Reader, domainReader io.Reader) (Validator, error)

Crates a Validator with the given readers Params:

posReader Reader to load the valid parts of speech from
domainReader Reader to load the valid subject domains

Returns:

An initialized Validator