master

package
v0.5.0-alpha.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jan 13, 2025 License: Apache-2.0 Imports: 65 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	LabelFeedbackType = "feedback_type"
	LabelStep         = "step"
	LabelData         = "data"
)
View Source 
const (
	EOF            = int64(0)
	UserStream     = int64(-1)
	ItemStream     = int64(-2)
	FeedbackStream = int64(-3)
)
View Source 
const (
	PositiveFeedbackRate = "PositiveFeedbackRate"

	TaskFindItemNeighbors      = "Find neighbors of items"
	TaskFindUserNeighbors      = "Find neighbors of users"
	TaskFitRankingModel        = "Fit collaborative filtering model"
	TaskFitClickModel          = "Fit click-through rate prediction model"
	TaskSearchRankingModel     = "Search collaborative filtering  model"
	TaskSearchClickModel       = "Search click-through rate prediction model"
	TaskCacheGarbageCollection = "Collect garbage in cache"
)
View Source 
const (
	ModelFile = "model.bin"
)

Variables

View Source 
var (
	LoadDatasetStepSecondsVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "load_dataset_step_seconds",
	}, []string{LabelStep})
	LoadDatasetTotalSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "load_dataset_total_seconds",
	})
	FindUserNeighborsSecondsVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_user_neighbors_seconds",
	}, []string{LabelStep})
	FindUserNeighborsTotalSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_user_neighbors_total_seconds",
	})
	FindItemNeighborsSecondsVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_item_neighbors_seconds",
	}, []string{"step"})
	FindItemNeighborsTotalSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_item_neighbors_total_seconds",
	})
	UpdateUserNeighborsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "update_user_neighbors_total",
	})
	UpdateItemNeighborsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "update_item_neighbors_total",
	})
	CacheScannedTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "cache_scanned_total",
	})
	CacheReclaimedTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "cache_reclaimed_total",
	})
	CacheScannedSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "cache_scanned_seconds",
	})

	CollaborativeFilteringFitSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_fit_seconds",
	})
	CollaborativeFilteringSearchSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_search_seconds",
	})
	CollaborativeFilteringNDCG10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_ndcg_10",
	})
	CollaborativeFilteringPrecision10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_precision_10",
	})
	CollaborativeFilteringRecall10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_recall_10",
	})
	CollaborativeFilteringSearchPrecision10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_search_precision_10",
	})
	RankingFitSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_fit_seconds",
	})
	RankingSearchSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_search_seconds",
	})
	RankingPrecision = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_precision",
	})
	RankingRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_recall",
	})
	RankingAUC = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_auc",
	})
	RankingSearchPrecision = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_search_precision",
	})
	UserNeighborIndexRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "user_neighbor_index_recall",
	})
	ItemNeighborIndexRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "item_neighbor_index_recall",
	})

	UsersTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "users_total",
	})
	ActiveUsersTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "active_users_total",
	})
	InactiveUsersTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "inactive_users_total",
	})
	ItemsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "items_total",
	})
	ActiveItemsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "active_items_total",
	})
	InactiveItemsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "inactive_items_total",
	})
	UserLabelsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "user_labels_total",
	})
	ItemLabelsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "item_labels_total",
	})
	FeedbacksTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "feedbacks_total",
	})
	ImplicitFeedbacksTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "implicit_feedbacks_total",
	})
	PositiveFeedbacksTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "positive_feedbacks_total",
	})
	NegativeFeedbackTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "negative_feedbacks_total",
	})
	MemoryInUseBytesVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "memory_inuse_bytes",
	}, []string{LabelData})
)

Functions

This section is empty.

Types

type CacheGarbageCollectionTask added in v0.4.6 

type CacheGarbageCollectionTask struct {
	*Master
}

func NewCacheGarbageCollectionTask added in v0.4.6 

func NewCacheGarbageCollectionTask(m *Master) *CacheGarbageCollectionTask

type DualDictionaryCentroidVector added in v0.4.2 

type DualDictionaryCentroidVector struct {
	// contains filtered or unexported fields
}

func (*DualDictionaryCentroidVector) Distance added in v0.4.2 

func (d *DualDictionaryCentroidVector) Distance(vector search.Vector) float32

type DualDictionaryVector added in v0.4.2 

type DualDictionaryVector struct {
	// contains filtered or unexported fields
}

func NewDualDictionaryVector added in v0.4.2 

func NewDualDictionaryVector(
	indices1 []int32, values1 []float32,
	indices2 []int32, values2 []float32,
	terms []string, isHidden bool) *DualDictionaryVector

func (*DualDictionaryVector) Centroid added in v0.4.2 

func (v *DualDictionaryVector) Centroid(vectors []search.Vector, indices []int32) search.CentroidVector

func (*DualDictionaryVector) Distance added in v0.4.2 

func (v *DualDictionaryVector) Distance(vector search.Vector) float32

func (*DualDictionaryVector) IsHidden added in v0.4.2 

func (v *DualDictionaryVector) IsHidden() bool

func (*DualDictionaryVector) Terms added in v0.4.2 

func (v *DualDictionaryVector) Terms() []string

type DualVectors added in v0.4.2 

type DualVectors struct {
	// contains filtered or unexported fields
}

func NewDualVectors added in v0.4.2 

func NewDualVectors(first, second *Vectors) *DualVectors

func (*DualVectors) Distance added in v0.4.2 

func (v *DualVectors) Distance(i, j int) float32

func (*DualVectors) Neighbors added in v0.4.2 

func (v *DualVectors) Neighbors(i int) []int32

type DumpStats 

type DumpStats struct {
	Users    int
	Items    int
	Feedback int
	Duration time.Duration
}

type Feedback 

type Feedback struct {
	FeedbackType string
	UserId       string
	Item         data.Item
	Timestamp    time.Time
	Comment      string
}

type FindItemNeighborsTask added in v0.4.6 

type FindItemNeighborsTask struct {
	*Master
	// contains filtered or unexported fields
}

FindItemNeighborsTask updates neighbors of items.

func NewFindItemNeighborsTask added in v0.4.6 

func NewFindItemNeighborsTask(m *Master) *FindItemNeighborsTask

type FindUserNeighborsTask added in v0.4.6 

type FindUserNeighborsTask struct {
	*Master
	// contains filtered or unexported fields
}

FindUserNeighborsTask updates neighbors of users.

func NewFindUserNeighborsTask added in v0.4.6 

func NewFindUserNeighborsTask(m *Master) *FindUserNeighborsTask

type FitClickModelTask added in v0.4.6 

type FitClickModelTask struct {
	*Master
	// contains filtered or unexported fields
}

FitClickModelTask fits click model using latest data. After model fitted, following states are changed: 1. Click model version are increased. 2. Click model score are updated. 3. Click model, version and score are persisted to local cache.

func NewFitClickModelTask added in v0.4.6 

func NewFitClickModelTask(m *Master) *FitClickModelTask

type FitRankingModelTask added in v0.4.6 

type FitRankingModelTask struct {
	*Master
	// contains filtered or unexported fields
}

func NewFitRankingModelTask added in v0.4.6 

func NewFitRankingModelTask(m *Master) *FitRankingModelTask

type LocalCache 

type LocalCache struct {
	RankingModelName    string
	RankingModelVersion int64
	RankingModel        ranking.MatrixFactorization
	RankingModelScore   ranking.Score
	ClickModelVersion   int64
	ClickModelScore     click.Score
	ClickModel          click.FactorizationMachine
	// contains filtered or unexported fields
}

LocalCache is local cache for the master node.

func LoadLocalCache 

func LoadLocalCache(path string) (*LocalCache, error)

LoadLocalCache loads local cache from a file. If the ranking model is invalid, RankingModel == nil. If the click model is invalid, ClickModel == nil.

func (*LocalCache) GetFilePath 

func (c *LocalCache) GetFilePath(file string) string

func (*LocalCache) WriteLocalCache 

func (c *LocalCache) WriteLocalCache() error

WriteLocalCache writes local cache to a file.

type Master 

type Master struct {
	protocol.UnimplementedMasterServer
	server.RestServer
	// contains filtered or unexported fields
}

Master is the master node.

func NewMaster 

func NewMaster(cfg *config.Config, cacheFile string, managedMode bool) *Master

NewMaster creates a master node.

func (*Master) CreateWebService 

func (m *Master) CreateWebService()

func (*Master) GetClickModel added in v0.2.2 

func (m *Master) GetClickModel(version *protocol.VersionInfo, sender protocol.Master_GetClickModelServer) error

GetClickModel returns latest click model.

func (*Master) GetItem 

func (m *Master) GetItem(score cache.Score) (any, error)

func (*Master) GetMeta 

func (m *Master) GetMeta(ctx context.Context, nodeInfo *protocol.NodeInfo) (*protocol.Meta, error)

GetMeta returns latest configuration.

func (*Master) GetRankingModel 

func (m *Master) GetRankingModel(version *protocol.VersionInfo, sender protocol.Master_GetRankingModelServer) error

GetRankingModel returns latest ranking model.

func (*Master) GetUser 

func (m *Master) GetUser(score cache.Score) (any, error)

func (*Master) LoadDataFromDatabase added in v0.2.6 

func (m *Master) LoadDataFromDatabase(
	ctx context.Context,
	database data.Database,
	posFeedbackTypes, readTypes []string,
	itemTTL, positiveFeedbackTTL uint,
	evaluator *OnlineEvaluator,
	nonPersonalizedRecommenders []*logics.NonPersonalized,
) (rankingDataset *ranking.DataSet, clickDataset *click.Dataset, err error)

LoadDataFromDatabase loads dataset from data store.

func (*Master) LoginFilter added in v0.4.5 

func (m *Master) LoginFilter(req *restful.Request, resp *restful.Response, chain *restful.FilterChain)

func (*Master) PushProgress 

func (m *Master) PushProgress(
	_ context.Context,
	in *protocol.PushProgressRequest) (*protocol.PushProgressResponse, error)

func (*Master) RunManagedTasksLoop added in v0.4.9 

func (m *Master) RunManagedTasksLoop()

func (*Master) RunPrivilegedTasksLoop added in v0.2.4 

func (m *Master) RunPrivilegedTasksLoop()

func (*Master) RunRagtagTasksLoop added in v0.2.4 

func (m *Master) RunRagtagTasksLoop()

RunRagtagTasksLoop searches optimal recommendation model in background. It never modifies variables other than rankingModelSearcher, clickSearchedModel and clickSearchedScore.

func (*Master) Serve 

func (m *Master) Serve()

Serve starts the master node.

func (*Master) SetOneMode added in v0.4.9 

func (m *Master) SetOneMode(workerScheduleHandler http.HandlerFunc)

func (*Master) Shutdown added in v0.4.6 

func (m *Master) Shutdown()

func (*Master) StartHttpServer 

func (m *Master) StartHttpServer()

type OnlineEvaluator added in v0.4.3 

type OnlineEvaluator struct {
	ReadFeedbacks      []map[int32]mapset.Set[int32]
	PositiveFeedbacks  map[string][]lo.Tuple3[int32, int32, time.Time]
	ReverseIndex       map[lo.Tuple2[int32, int32]]time.Time
	EvaluateDays       int
	TruncatedDateToday time.Time
}

func NewOnlineEvaluator added in v0.4.3 

func NewOnlineEvaluator() *OnlineEvaluator

func (*OnlineEvaluator) Evaluate added in v0.4.3 

func (evaluator *OnlineEvaluator) Evaluate() []cache.TimeSeriesPoint

func (*OnlineEvaluator) Positive added in v0.4.3 

func (evaluator *OnlineEvaluator) Positive(feedbackType string, userIndex, itemIndex int32, timestamp time.Time)

func (*OnlineEvaluator) Read added in v0.4.3 

func (evaluator *OnlineEvaluator) Read(userIndex, itemIndex int32, timestamp time.Time)

type ScheduleState added in v0.4.9 

type ScheduleState struct {
	IsRunning   bool      `json:"is_running"`
	SearchModel bool      `json:"search_model"`
	StartTime   time.Time `json:"start_time"`
}

type ScoreUser added in v0.4.8 

type ScoreUser struct {
	data.User
	Score float64
}

type ScoredItem added in v0.4.8 

type ScoredItem struct {
	data.Item
	Score float64
}

type SearchClickModelTask added in v0.4.6 

type SearchClickModelTask struct {
	*Master
	// contains filtered or unexported fields
}

SearchClickModelTask searches best hyper-parameters for factorization machines. It requires read lock on the click dataset.

func NewSearchClickModelTask added in v0.4.6 

func NewSearchClickModelTask(m *Master) *SearchClickModelTask

type SearchRankingModelTask added in v0.4.6 

type SearchRankingModelTask struct {
	*Master
	// contains filtered or unexported fields
}

SearchRankingModelTask searches best hyper-parameters for ranking models. It requires read lock on the ranking dataset.

func NewSearchRankingModelTask added in v0.4.6 

func NewSearchRankingModelTask(m *Master) *SearchRankingModelTask

type SinglePageAppFileSystem 

type SinglePageAppFileSystem struct {
	// contains filtered or unexported fields
}

SinglePageAppFileSystem is the file system for single page app.

func (*SinglePageAppFileSystem) Open 

func (fs *SinglePageAppFileSystem) Open(name string) (http.File, error)

Open index.html if required file not exists.

type Status 

type Status struct {
	BinaryVersion           string
	NumServers              int
	NumWorkers              int
	NumUsers                int
	NumItems                int
	NumUserLabels           int
	NumItemLabels           int
	NumTotalPosFeedback     int
	NumValidPosFeedback     int
	NumValidNegFeedback     int
	PopularItemsUpdateTime  time.Time
	LatestItemsUpdateTime   time.Time
	MatchingModelFitTime    time.Time
	MatchingModelScore      ranking.Score
	RankingModelFitTime     time.Time
	RankingModelScore       click.Score
	UserNeighborIndexRecall float32
	ItemNeighborIndexRecall float32
	MatchingIndexRecall     float32
}

type Task added in v0.2.4 

type Task interface {
	// contains filtered or unexported methods
}

type User 

type User struct {
	data.User
	LastActiveTime time.Time
	LastUpdateTime time.Time
}

type UserInfo 

type UserInfo struct {
	Name       string `json:"name"`
	FamilyName string `json:"family_name"`
	GivenName  string `json:"given_name"`
	MiddleName string `json:"middle_name"`
	NickName   string `json:"nickname"`
	Picture    string `json:"picture"`
	UpdatedAt  string `json:"updated_at"`
	Email      string `json:"email"`
	Verified   bool   `json:"email_verified"`
	AuthType   string `json:"auth_type"`
}

type UserIterator 

type UserIterator struct {
	Cursor string
	Users  []User
}

type Vectors added in v0.4.2 

type Vectors struct {
	// contains filtered or unexported fields
}

func NewVectors added in v0.4.2 

func NewVectors(connections, connected [][]int32, weights []float32) *Vectors

func (*Vectors) Distance added in v0.4.2 

func (v *Vectors) Distance(i, j int) float32

func (*Vectors) Neighbors added in v0.4.2 

func (v *Vectors) Neighbors(i int) []int32

type VectorsInterface added in v0.4.2 

type VectorsInterface interface {
	Distance(i, j int) float32
	Neighbors(i int) []int32
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.