es

package
v1.3.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 26, 2019 License: MIT Imports: 30 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	LANGUAGES_MAX_FAILURE      = 3
	DOCUMENT_MAX_FAILIRE_RATIO = 0.1
)

Variables

This section is empty.

Functions

func CollectionsScopeByContentUnit added in v0.7.7 

func CollectionsScopeByContentUnit(mdb *sql.DB, cuUID string) ([]string, error)

func CollectionsScopeByFile added in v0.7.7 

func CollectionsScopeByFile(mdb *sql.DB, fileUID string) ([]string, error)

func ConvertDocx 

func ConvertDocx(db *sql.DB) error

func DocText added in v1.2.4 

func DocText(uid string) (string, error)

func DumpDB added in v0.7.7 

func DumpDB(mdb *sql.DB, title string) error

func DumpIndexes added in v0.7.7 

func DumpIndexes(esc *elastic.Client, title string, resultType string) error

func IndexAliasName added in v1.0.5 

func IndexAliasName(namespace string, name string, lang string) string

func IndexName 

func IndexName(namespace string, name string, lang string, date string) string

func InitConfigFolder added in v0.8.9 

func InitConfigFolder(configKey string, value *string) (string, error)

func InitVars added in v0.7.7 

func InitVars()

func IsWindows added in v0.8.9 

func IsWindows() bool

func KeyIValues added in v1.0.5 

func KeyIValues(t string, uids []string) []interface{}

func KeyValues added in v1.0.5 

func KeyValues(t string, uids []string) []string

func KeyValuesToValues added in v1.0.5 

func KeyValuesToValues(t string, typedUIDs []string) ([]string, error)

func ParseDocx 

func ParseDocx(docxPath string) (string, error)

func Prepare added in v1.2.4 

func Prepare(uids []string) (error, bool, map[string]int)

func ProdAliasedIndexDate added in v1.0.5 

func ProdAliasedIndexDate(esc *elastic.Client) (error, string)

func SourcesFolder added in v0.8.9 

func SourcesFolder() (string, error)

func Suffixes added in v1.0.5 

func Suffixes(title string) []string

func SwitchAliasToCurrentIndex added in v1.0.5 

func SwitchAliasToCurrentIndex(namespace string, name string, date string, esc *elastic.Client) error

func SwitchProdAliasToCurrentIndex added in v1.0.5 

func SwitchProdAliasToCurrentIndex(date string, esc *elastic.Client) error

Types

type BaseIndex added in v0.6.7 

type BaseIndex struct {
	// contains filtered or unexported fields
}

func (*BaseIndex) CreateIndex added in v0.6.7 

func (index *BaseIndex) CreateIndex() error

func (*BaseIndex) DeleteIndex added in v0.6.7 

func (index *BaseIndex) DeleteIndex() error

func (*BaseIndex) FilterByResultTypeQuery added in v1.0.5 

func (index *BaseIndex) FilterByResultTypeQuery(resultType string) *elastic.BoolQuery

func (*BaseIndex) IndexName added in v1.3.4 

func (index *BaseIndex) IndexName(lang string) string

func (*BaseIndex) RefreshIndex added in v0.6.7 

func (index *BaseIndex) RefreshIndex() error

func (*BaseIndex) RefreshIndexByLang added in v0.6.7 

func (index *BaseIndex) RefreshIndexByLang(lang string) error

func (*BaseIndex) RemoveFromIndexQuery added in v1.0.5 

func (index *BaseIndex) RemoveFromIndexQuery(elasticScope elastic.Query) (map[string][]string, *IndexErrors)

func (*BaseIndex) ResultType added in v1.0.5 

func (index *BaseIndex) ResultType() string

func (*BaseIndex) Scroll added in v1.3.4 

func (index *BaseIndex) Scroll(indexName string, elasticScope elastic.Query) ([]ScrollResult, error)

type BlogIndex added in v1.1.8 

type BlogIndex struct {
	BaseIndex
	Progress uint64
}

func MakeBlogIndex added in v1.1.8 

func MakeBlogIndex(namespace string, indexDate string, db *sql.DB, esc *elastic.Client) *BlogIndex

func (*BlogIndex) AddToIndex added in v1.3.4 

func (index *BlogIndex) AddToIndex(scope Scope, removedUIDs []string) error

func (*BlogIndex) ReindexAll added in v1.1.8 

func (index *BlogIndex) ReindexAll() error

func (*BlogIndex) RemoveFromIndex added in v1.3.4 

func (index *BlogIndex) RemoveFromIndex(scope Scope) (map[string][]string, error)

type Classification 

type Classification struct {
	MDB_UID            string `json:"mdb_uid"`
	Name               string `json:"name,omitempty"`
	NameSuggest        string `json:"name_suggest,omitempty"`
	Description        string `json:"description,omitempty"`
	DescriptionSuggest string `json:"description_suggest,omitempty"`
	Type               string `json:"classification_type"`
}

type ClassificationIntent added in v0.8.9 

type ClassificationIntent struct {
	// Fields from result.
	ResultType string `json:"result_type"`
	MDB_UID    string `json:"mdb_uid"`
	Title      string `json:"title"`

	// Intent fields.
	ContentType    string                    `json:"content_type"`
	Exist          bool                      `json:"exist"`
	Score          *float64                  `json:"score,omitempty"`
	Explanation    elastic.SearchExplanation `json:"explanation,omitempty"`
	MaxScore       *float64                  `json:"max_score,omitempty"`
	MaxExplanation elastic.SearchExplanation `json:"max_explanation,omitempty"`
}

type Collection 

type Collection struct {
	MDB_UID                  string      `json:"mdb_uid"`
	TypedUIDs                []string    `json:"typed_uids"`
	Name                     string      `json:"name"`
	Description              string      `json:"description"`
	ContentType              string      `json:"content_type"`
	ContentUnitsContentTypes []string    `json:"content_units_content_types,omitempty"`
	EffectiveDate            *utils.Date `json:"effective_date"`
	OriginalLanguage         string      `json:"original_language,omitempty"`
}

type CollectionsIndex added in v0.6.7 

type CollectionsIndex struct {
	BaseIndex
}

func MakeCollectionsIndex added in v0.6.9 

func MakeCollectionsIndex(namespace string, indexDate string, db *sql.DB, esc *elastic.Client) *CollectionsIndex

func (*CollectionsIndex) AddToIndex added in v1.3.4 

func (index *CollectionsIndex) AddToIndex(scope Scope, removedUIDs []string) error

func (*CollectionsIndex) ReindexAll added in v0.6.9 

func (index *CollectionsIndex) ReindexAll() error

func (*CollectionsIndex) RemoveFromIndex added in v1.3.4 

func (index *CollectionsIndex) RemoveFromIndex(scope Scope) (map[string][]string, error)

type ContentUnit 

type ContentUnit struct {
	MDB_UID                 string      `json:"mdb_uid"`
	TypedUIDs               []string    `json:"typed_uids"`
	Name                    string      `json:"name,omitempty"`
	Description             string      `json:"description,omitempty"`
	ContentType             string      `json:"content_type"`
	CollectionsContentTypes []string    `json:"collections_content_types,omitempty"`
	EffectiveDate           *utils.Date `json:"effective_date,omitempty"`
	Duration                uint64      `json:"duration,omitempty"`
	OriginalLanguage        string      `json:"original_language,omitempty"`
	Translations            []string    `json:"translations,omitempty"`
	Tags                    []string    `json:"tags,omitempty"`
	Sources                 []string    `json:"sources,omitempty"`
	Authors                 []string    `json:"authors,omitempty"`
	Persons                 []string    `json:"persons,omitempty"`
	Transcript              string      `json:"transcript,omitempty"`
}

type ContentUnitsIndex added in v0.6.7 

type ContentUnitsIndex struct {
	BaseIndex
	Progress uint64
}

func MakeContentUnitsIndex added in v0.6.7 

func MakeContentUnitsIndex(namespace string, indexDate string, db *sql.DB, esc *elastic.Client) *ContentUnitsIndex

func (*ContentUnitsIndex) AddToIndex added in v1.3.4 

func (index *ContentUnitsIndex) AddToIndex(scope Scope, removedUIDs []string) error

func (*ContentUnitsIndex) ReindexAll added in v0.6.7 

func (index *ContentUnitsIndex) ReindexAll() error

func (*ContentUnitsIndex) RemoveFromIndex added in v1.3.4 

func (index *ContentUnitsIndex) RemoveFromIndex(scope Scope) (map[string][]string, error)

type DocumentError added in v1.3.4 

type DocumentError struct {
	Error error
	Count int
}

type EffectiveDate added in v0.6.9 

type EffectiveDate struct {
	EffectiveDate *utils.Date `json:"effective_date"`
}

type File 

type File struct {
	MDB_UID  string      `json:"mdb_uid"`
	Name     string      `json:"name"`
	Size     uint64      `json:"size"`
	URL      string      `json:"url"`
	Secure   int         `json:"secure"`
	FilmDate *utils.Date `json:"film_date"`
	Duration float64     `json:"duration,omitempty"`
	Language string      `json:"language,omitempty"`
	MimeType string      `json:"mimetype,omitempty"`
	Type     string      `json:"type,omitempty"`
	SubType  string      `json:"subtype,omitempty"`
}

type Index added in v0.6.7 

type Index interface {
	ReindexAll() error
	RemoveFromIndex(scope Scope) (map[string][]string, error)
	AddToIndex(scope Scope, removedUIDs []string) error
	CreateIndex() error
	DeleteIndex() error
	RefreshIndex() error
	ResultType() string
	IndexName(language string) string
}

type IndexData added in v0.6.7 

type IndexData struct {
	DB      *sql.DB
	Sources map[string][]string
	Tags    map[string][]string
	// Persons      map[string][]string
	// Translations map[string][][]string
	MediaLanguages map[string][]string
	Transcripts    map[string]map[string][]string
}

func MakeIndexData added in v0.7.7 

func MakeIndexData(db *sql.DB, sqlScope string) (*IndexData, error)

type IndexErrors added in v1.3.4 

type IndexErrors struct {
	Error           error
	LanguageErrors  map[string][]error
	DocumentsCount  map[string]int
	DocumentsErrors map[string][]DocumentError
}

func MakeIndexErrors added in v1.3.4 

func MakeIndexErrors() *IndexErrors

func (*IndexErrors) CheckErrors added in v1.3.4 

func (indexErrors *IndexErrors) CheckErrors(languagesMaxFailure int, documentsMaxFailure float32, index string) error

func (*IndexErrors) DocumentError added in v1.3.4 

func (indexErrors *IndexErrors) DocumentError(language string, e error, info string) *IndexErrors

func (*IndexErrors) DocumentErrorCount added in v1.3.4 

func (indexErrors *IndexErrors) DocumentErrorCount(language string, e error, info string, count int) *IndexErrors

func (*IndexErrors) FailedLanguages added in v1.3.4 

func (indexErrors *IndexErrors) FailedLanguages() string

func (*IndexErrors) Join added in v1.3.4 

func (indexErrors *IndexErrors) Join(other *IndexErrors, info string) *IndexErrors

func (*IndexErrors) LanguageError added in v1.3.4 

func (indexErrors *IndexErrors) LanguageError(language string, e error, info string) *IndexErrors

func (*IndexErrors) LanguagesError added in v1.3.4 

func (indexErrors *IndexErrors) LanguagesError() error

func (*IndexErrors) Print added in v1.3.4 

func (indexErrors *IndexErrors) Print()

func (*IndexErrors) PrintDocumentsErrors added in v1.3.4 

func (indexErrors *IndexErrors) PrintDocumentsErrors(info string)

func (*IndexErrors) PrintLanguageErrors added in v1.3.4 

func (indexErrors *IndexErrors) PrintLanguageErrors(info string)

func (*IndexErrors) PrintShort added in v1.3.4 

func (indexErrors *IndexErrors) PrintShort()

func (*IndexErrors) SetError added in v1.3.4 

func (indexErrors *IndexErrors) SetError(err error) *IndexErrors

func (*IndexErrors) Wrap added in v1.3.4 

func (indexErrors *IndexErrors) Wrap(info string) *IndexErrors

type Indexer added in v0.6.7 

type Indexer struct {
	// contains filtered or unexported fields
}

func MakeFakeIndexer added in v0.7.0 

func MakeFakeIndexer(mdb *sql.DB, esc *elastic.Client) (*Indexer, error)

func MakeIndexer added in v0.6.7 

func MakeIndexer(namespace string, date string, names []string, mdb *sql.DB, esc *elastic.Client) (*Indexer, error)

Receives namespace and list of indexes names.

func MakeProdIndexer added in v0.6.7 

func MakeProdIndexer(date string, mdb *sql.DB, esc *elastic.Client) (*Indexer, error)

func (*Indexer) BlogPostUpdate added in v1.1.8 

func (indexer *Indexer) BlogPostUpdate(id string) error

func (*Indexer) CollectionUpdate added in v0.6.7 

func (indexer *Indexer) CollectionUpdate(uid string) error

Set of MDB event handlers to incrementally change all indexes.

func (*Indexer) ContentUnitUpdate added in v0.6.7 

func (indexer *Indexer) ContentUnitUpdate(uid string) error

func (*Indexer) CreateIndexes added in v0.6.7 

func (indexer *Indexer) CreateIndexes() error

func (*Indexer) DeleteIndexes added in v0.6.7 

func (indexer *Indexer) DeleteIndexes() error

func (*Indexer) FileUpdate added in v0.6.7 

func (indexer *Indexer) FileUpdate(uid string) error

func (*Indexer) PersonUpdate added in v0.6.7 

func (indexer *Indexer) PersonUpdate(uid string) error

func (*Indexer) PublisherUpdate added in v0.6.7 

func (indexer *Indexer) PublisherUpdate(uid string) error

func (*Indexer) RefreshAll added in v0.6.7 

func (indexer *Indexer) RefreshAll() error

func (*Indexer) ReindexAll added in v0.6.7 

func (indexer *Indexer) ReindexAll() error

func (*Indexer) SourceUpdate added in v0.6.7 

func (indexer *Indexer) SourceUpdate(uid string) error

func (*Indexer) TagUpdate added in v0.6.7 

func (indexer *Indexer) TagUpdate(uid string) error

func (*Indexer) TweetUpdate added in v1.1.8 

func (indexer *Indexer) TweetUpdate(tid string) error

func (*Indexer) Update added in v1.3.2 

func (indexer *Indexer) Update(scope Scope) error

type OffsetLimitJob added in v1.0.5 

type OffsetLimitJob struct {
	Offset int
	Limit  int
	Total  int
}

type Result added in v1.0.5 

type Result struct {
	// Document type.
	ResultType string `json:"result_type"`

	MDB_UID      string   `json:"mdb_uid"`
	TypedUids    []string `json:"typed_uids"`
	FilterValues []string `json:"filter_values"`

	// Result content fields.
	Title       string `json:"title"`
	Description string `json:"description,omitempty"`
	Content     string `json:"content,omitempty"`

	EffectiveDate *utils.Date `json:"effective_date,omitempty"`

	// Suggest field for autocomplete.
	TitleSuggest []string `json:"title_suggest"`
}

For full description see make.py RESULTS TEMPLATE.

func (*Result) ToDebugString added in v1.1.3 

func (result *Result) ToDebugString() string

type ResultType added in v1.0.5 

type ResultType struct {
	ResultType string `json:"result_type"`
}

type Scope added in v0.6.7 

type Scope struct {
	ContentUnitUID string
	FileUID        string
	CollectionUID  string
	TagUID         string
	SourceUID      string
	PersonUID      string
	PublisherUID   string
	TweetTID       string
	BlogPostWPID   string
}

type ScrollResult added in v1.3.4 

type ScrollResult struct {
	MdbUid     string
	Id         string
	ResultType string
}

type Source added in v0.8.9 

type Source struct {
	MDB_UID string `json:"mdb_uid"`
	Name    string `json:"name"`

	// Deprecated fields (since we use 'Result Template' in order to index the sources):
	Description string   `json:"description"`
	Content     string   `json:"content"`
	Sources     []string `json:"sources"`
	Authors     []string `json:"authors"`
	PathNames   []string `json:"path_names"`
	FullName    []string `json:"full_name"`
}

type SourcesIndex added in v0.8.9 

type SourcesIndex struct {
	BaseIndex
	Progress uint64
}

func MakeSourcesIndex added in v0.8.9 

func MakeSourcesIndex(namespace string, indexDate string, db *sql.DB, esc *elastic.Client) *SourcesIndex

func (*SourcesIndex) AddToIndex added in v1.3.4 

func (index *SourcesIndex) AddToIndex(scope Scope, removedUIDs []string) error

func (*SourcesIndex) ReindexAll added in v0.8.9 

func (index *SourcesIndex) ReindexAll() error

func (*SourcesIndex) RemoveFromIndex added in v1.3.4 

func (index *SourcesIndex) RemoveFromIndex(scope Scope) (map[string][]string, error)

type TagsIndex added in v1.0.5 

type TagsIndex struct {
	BaseIndex
}

func MakeTagsIndex added in v1.0.5 

func MakeTagsIndex(namespace string, indexDate string, db *sql.DB, esc *elastic.Client) *TagsIndex

func (*TagsIndex) AddToIndex added in v1.3.4 

func (index *TagsIndex) AddToIndex(scope Scope, removedUIDs []string) error

func (*TagsIndex) ReindexAll added in v1.0.5 

func (index *TagsIndex) ReindexAll() error

func (*TagsIndex) RemoveFromIndex added in v1.3.4 

func (index *TagsIndex) RemoveFromIndex(scope Scope) (map[string][]string, error)

type TweeterIndex added in v1.1.8 

type TweeterIndex struct {
	BaseIndex
	Progress uint64
}

func MakeTweeterIndex added in v1.1.8 

func MakeTweeterIndex(namespace string, indexDate string, db *sql.DB, esc *elastic.Client) *TweeterIndex

func (*TweeterIndex) AddToIndex added in v1.3.4 

func (index *TweeterIndex) AddToIndex(scope Scope, removedUIDs []string) error

func (*TweeterIndex) ReindexAll added in v1.1.8 

func (index *TweeterIndex) ReindexAll() error

func (*TweeterIndex) RemoveFromIndex added in v1.3.4 

func (index *TweeterIndex) RemoveFromIndex(scope Scope) (map[string][]string, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.