repo

Documentation

Index

• func NormalizeTo(input, form string) string
• type DictPhonemizerRepository
  • func NewDictPhonemizerRepository(di *DependencyInjection) *DictPhonemizerRepository
  • func (r *DictPhonemizerRepository) LoadLanguage(isReverse bool, lang string)
  • func (r *DictPhonemizerRepository) LookupWords(isReverse bool, lang, word string) (ret []map[uint64]string)
• type HashtronPhonemizerRepository
  • func NewHashtronPhonemizerRepository(di *DependencyInjection) *HashtronPhonemizerRepository
  • func (r *HashtronPhonemizerRepository) CheckWord(isReverse bool, lang, word, ipa string) bool
  • func (r *HashtronPhonemizerRepository) CleanWord(isReverse bool, lang, word string) (ret string)
  • func (r *HashtronPhonemizerRepository) LoadLanguage(isReverse bool, lang string)
  • func (r *HashtronPhonemizerRepository) PhonemizeWords(isReverse bool, lang string, word string) (ret []map[uint64]string)
• type IDictPhonemizerRepository
• type IHashtronPhonemizerRepository
• type IPrePhonWordStepsRepository
• type ISpaceSplitterRepository
• type IWordCachingRepository
• type PrePhonWordStep
• type PrePhonWordStepsRepository
  • func NewPrePhonWordStepsRepository(di *DependencyInjection) *PrePhonWordStepsRepository
  • func (p *PrePhonWordStepsRepository) LoadLanguage(isReverse bool, lang string)
  • func (s *PrePhonWordStepsRepository) PrePhonemizeWord(isReverse bool, lang string, word string) string
• type SpaceSplitterRepository
  • func NewSpaceSplitterRepository(di *DependencyInjection) *SpaceSplitterRepository
  • func (p *SpaceSplitterRepository) LoadLanguage(isReverse bool, lang string)
  • func (s *SpaceSplitterRepository) Split(sentence string) []string
  • func (s *SpaceSplitterRepository) SplitLang(isReverse bool, lang, sentence string) []string
• type WordCachingRepository
  • func NewWordCachingRepository(di *DependencyInjection) *WordCachingRepository
  • func (r WordCachingRepository) HashWord(isReverse bool, lang, word string) uint64
  • func (r WordCachingRepository) LoadWord(hash uint64) (word map[uint64]string)
  • func (r WordCachingRepository) StoreWord(value map[uint64]string, hash uint64)

Functions

func NormalizeTo

func NormalizeTo(input, form string) string

Function to normalize text to NFC

Types

type DictPhonemizerRepository

type DictPhonemizerRepository struct {
	// contains filtered or unexported fields
}

func NewDictPhonemizerRepository

func NewDictPhonemizerRepository(di *DependencyInjection) *DictPhonemizerRepository

func (*DictPhonemizerRepository) LoadLanguage

func (r *DictPhonemizerRepository) LoadLanguage(isReverse bool, lang string)

func (*DictPhonemizerRepository) LookupWords

func (r *DictPhonemizerRepository) LookupWords(isReverse bool, lang, word string) (ret []map[uint64]string)

type HashtronPhonemizerRepository

type HashtronPhonemizerRepository struct {
	// contains filtered or unexported fields
}

func NewHashtronPhonemizerRepository

func NewHashtronPhonemizerRepository(di *DependencyInjection) *HashtronPhonemizerRepository

func (*HashtronPhonemizerRepository) CheckWord

func (r *HashtronPhonemizerRepository) CheckWord(isReverse bool, lang, word, ipa string) bool

func (*HashtronPhonemizerRepository) CleanWord

func (r *HashtronPhonemizerRepository) CleanWord(isReverse bool, lang, word string) (ret string)

func (*HashtronPhonemizerRepository) LoadLanguage

func (r *HashtronPhonemizerRepository) LoadLanguage(isReverse bool, lang string)

func (*HashtronPhonemizerRepository) PhonemizeWords

func (r *HashtronPhonemizerRepository) PhonemizeWords(isReverse bool, lang string, word string) (ret []map[uint64]string)

func (r *HashtronPhonemizerRepository) PhonemizeWord(isReverse bool, lang string, word string) (ret map[uint64]string) {
	var reverse string
	if isReverse {
		reverse = "_reverse"
	}
	r.LoadLanguage(isReverse, lang)

	r.mut.RLock()
	mapLangIsNil := r.lang.Slice(isReverse, lang) == nil
	r.mut.RUnlock()
	if mapLangIsNil {
		m := make(map[uint64]string)
		return m
	}

	r.mut.RLock()
	histogram := r.lang.Histogram(isReverse, lang)
	net := (*r.aregnets)[lang+reverse]
	r.mut.RUnlock()

	if net == nil {
		m := make(map[uint64]string)
		return m
	}

	for i := 0; i < 32; i++ {
		var input = phonemizer.AregSample{
			Src: word,
			Dst: fmt.Sprint(i),
		}
		r.mut.RLock()
		pred := net.Infer2(&input) == 1
		r.mut.RUnlock()

		if !pred {
			m := make(map[uint64]string)
			return m
		}
	}
	pred := true
	var out string
	for pred && len(out) < len(word)*2 {
		for _, val := range histogram {
			var input2 = phonemizer.AregSample{
				Src: word,
				Dst: out + val,
			}
			r.mut.RLock()
			pred = net.Infer2(&input2) == 1
			r.mut.RUnlock()
			//fmt.Println(word, out, val, pred)
			if pred {
				out += val
				break
			}
		}
	}
	//fmt.Println(word, out)
	m := make(map[uint64]string)
	hsh := murmur3hash(word + "\x00" + out)
	if hsh == 0 {
		hsh++
	}
	m[hsh] = out
	m[0] = word
	return m
}

type IDictPhonemizerRepository

type IDictPhonemizerRepository interface {
	LookupWords(isReverse bool, lang string, word string) []map[uint64]string
}

type IHashtronPhonemizerRepository

type IHashtronPhonemizerRepository interface {
	CleanWord(isReverse bool, lang, word string) string
	CheckWord(isReverse bool, lang, word, ipa string) bool
	PhonemizeWords(isReverse bool, lang string, word string) []map[uint64]string
}

type IPrePhonWordStepsRepository

type IPrePhonWordStepsRepository interface {
	PrePhonemizeWord(isReverse bool, lang string, word string) string
}

type ISpaceSplitterRepository

type ISpaceSplitterRepository interface {
	Split(string) []string
	SplitLang(bool, string, string) []string
}

type IWordCachingRepository

type IWordCachingRepository interface {
	HashWord(isReverse bool, lang, word string) uint64
	LoadWord(hash uint64) map[uint64]string
	StoreWord(one map[uint64]string, hash uint64)
}

type PrePhonWordStep

type PrePhonWordStep struct {
	Normalize string `json:"Normalize"`
	Trim      string `json:"Trim"`
	ToLower   bool   `json:"ToLower"`
}

type PrePhonWordStepsRepository

type PrePhonWordStepsRepository struct {
	// contains filtered or unexported fields
}

func NewPrePhonWordStepsRepository

func NewPrePhonWordStepsRepository(di *DependencyInjection) *PrePhonWordStepsRepository

func (*PrePhonWordStepsRepository) LoadLanguage

func (p *PrePhonWordStepsRepository) LoadLanguage(isReverse bool, lang string)

func (*PrePhonWordStepsRepository) PrePhonemizeWord

func (s *PrePhonWordStepsRepository) PrePhonemizeWord(isReverse bool, lang string, word string) string

type SpaceSplitterRepository

type SpaceSplitterRepository struct {
	// contains filtered or unexported fields
}

func NewSpaceSplitterRepository

func NewSpaceSplitterRepository(di *DependencyInjection) *SpaceSplitterRepository

func (*SpaceSplitterRepository) LoadLanguage

func (p *SpaceSplitterRepository) LoadLanguage(isReverse bool, lang string)

func (*SpaceSplitterRepository) Split

func (s *SpaceSplitterRepository) Split(sentence string) []string

func (*SpaceSplitterRepository) SplitLang

func (s *SpaceSplitterRepository) SplitLang(isReverse bool, lang, sentence string) []string

type WordCachingRepository

type WordCachingRepository struct {
	// contains filtered or unexported fields
}

func NewWordCachingRepository

func NewWordCachingRepository(di *DependencyInjection) *WordCachingRepository

func (WordCachingRepository) HashWord

func (r WordCachingRepository) HashWord(isReverse bool, lang, word string) uint64

func (WordCachingRepository) LoadWord

func (r WordCachingRepository) LoadWord(hash uint64) (word map[uint64]string)

func (WordCachingRepository) StoreWord

func (r WordCachingRepository) StoreWord(value map[uint64]string, hash uint64)