master

package
v0.4.6 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 16, 2022 License: Apache-2.0 Imports: 54 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	LabelFeedbackType = "feedback_type"
	LabelStep         = "step"
	LabelData         = "data"
)
View Source 
const (
	ServerNode = "Server"
	WorkerNode = "Worker"
)
View Source 
const (
	PositiveFeedbackRate = "PositiveFeedbackRate"

	TaskLoadDataset            = "Load dataset"
	TaskFindItemNeighbors      = "Find neighbors of items"
	TaskFindUserNeighbors      = "Find neighbors of users"
	TaskFitRankingModel        = "Fit collaborative filtering model"
	TaskFitClickModel          = "Fit click-through rate prediction model"
	TaskSearchRankingModel     = "Search collaborative filtering  model"
	TaskSearchClickModel       = "Search click-through rate prediction model"
	TaskCacheGarbageCollection = "Collect garbage in cache"
)

Variables

View Source 
var (
	LoadDatasetStepSecondsVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "load_dataset_step_seconds",
	}, []string{LabelStep})
	LoadDatasetTotalSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "load_dataset_total_seconds",
	})
	FindUserNeighborsSecondsVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_user_neighbors_seconds",
	}, []string{LabelStep})
	FindUserNeighborsTotalSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_user_neighbors_total_seconds",
	})
	FindItemNeighborsSecondsVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_item_neighbors_seconds",
	}, []string{"step"})
	FindItemNeighborsTotalSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_item_neighbors_total_seconds",
	})
	UpdateUserNeighborsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "update_user_neighbors_total",
	})
	UpdateItemNeighborsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "update_item_neighbors_total",
	})
	CacheScannedTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "cache_scanned_total",
	})
	CacheReclaimedTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "cache_reclaimed_total",
	})
	CacheScannedSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "cache_scanned_seconds",
	})

	CollaborativeFilteringFitSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_fit_seconds",
	})
	CollaborativeFilteringSearchSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_search_seconds",
	})
	CollaborativeFilteringNDCG10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_ndcg_10",
	})
	CollaborativeFilteringPrecision10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_precision_10",
	})
	CollaborativeFilteringRecall10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_recall_10",
	})
	CollaborativeFilteringSearchPrecision10 = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "collaborative_filtering_search_precision_10",
	})
	RankingFitSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_fit_seconds",
	})
	RankingSearchSeconds = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_search_seconds",
	})
	RankingPrecision = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_precision",
	})
	RankingRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_recall",
	})
	RankingAUC = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_auc",
	})
	RankingSearchPrecision = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_search_precision",
	})
	UserNeighborIndexRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "user_neighbor_index_recall",
	})
	ItemNeighborIndexRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "item_neighbor_index_recall",
	})

	UsersTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "users_total",
	})
	ActiveUsersTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "active_users_total",
	})
	InactiveUsersTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "inactive_users_total",
	})
	ItemsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "items_total",
	})
	ActiveItemsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "active_items_total",
	})
	InactiveItemsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "inactive_items_total",
	})
	UserLabelsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "user_labels_total",
	})
	ItemLabelsTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "item_labels_total",
	})
	FeedbacksTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "feedbacks_total",
	})
	ImplicitFeedbacksTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "implicit_feedbacks_total",
	})
	PositiveFeedbacksTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "positive_feedbacks_total",
	})
	NegativeFeedbackTotal = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "negative_feedbacks_total",
	})
	MemoryInuseBytesVec = promauto.NewGaugeVec(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "memory_inuse_bytes",
	}, []string{LabelData})
)

Functions

This section is empty.

Types

type DualDictionaryCentroidVector 

type DualDictionaryCentroidVector struct {
	// contains filtered or unexported fields
}

func (*DualDictionaryCentroidVector) Distance 

func (d *DualDictionaryCentroidVector) Distance(vector search.Vector) float32

type DualDictionaryVector 

type DualDictionaryVector struct {
	// contains filtered or unexported fields
}

func NewDualDictionaryVector 

func NewDualDictionaryVector(
	indices1 []int32, values1 []float32,
	indices2 []int32, values2 []float32,
	terms []string, isHidden bool) *DualDictionaryVector

func (*DualDictionaryVector) Centroid 

func (v *DualDictionaryVector) Centroid(vectors []search.Vector, indices []int32) search.CentroidVector

func (*DualDictionaryVector) Distance 

func (v *DualDictionaryVector) Distance(vector search.Vector) float32

func (*DualDictionaryVector) IsHidden 

func (v *DualDictionaryVector) IsHidden() bool

func (*DualDictionaryVector) Terms 

func (v *DualDictionaryVector) Terms() []string

type DualVectors 

type DualVectors struct {
	// contains filtered or unexported fields
}

func NewDualVectors 

func NewDualVectors(first, second *Vectors) *DualVectors

func (*DualVectors) Distance 

func (v *DualVectors) Distance(i, j int) float32

func (*DualVectors) Neighbors 

func (v *DualVectors) Neighbors(i int) []int32

type Feedback 

type Feedback struct {
	FeedbackType string
	UserId       string
	Item         data.Item
	Timestamp    time.Time
	Comment      string
}

type LocalCache 

type LocalCache struct {
	RankingModelName    string
	RankingModelVersion int64
	RankingModel        ranking.MatrixFactorization
	RankingModelScore   ranking.Score
	ClickModelVersion   int64
	ClickModelScore     click.Score
	ClickModel          click.FactorizationMachine
	// contains filtered or unexported fields
}

LocalCache is local cache for the master node.

func LoadLocalCache 

func LoadLocalCache(path string) (*LocalCache, error)

LoadLocalCache loads local cache from a file. If the ranking model is invalid, RankingModel == nil. If the click model is invalid, ClickModel == nil.

func (*LocalCache) WriteLocalCache 

func (c *LocalCache) WriteLocalCache() error

WriteLocalCache writes local cache to a file.

type Master 

type Master struct {
	protocol.UnimplementedMasterServer
	server.RestServer
	// contains filtered or unexported fields
}

Master is the master node.

func NewMaster 

func NewMaster(cfg *config.Config, cacheFile string) *Master

NewMaster creates a master node.

func (*Master) AuthFilter 

func (m *Master) AuthFilter(req *restful.Request, resp *restful.Response, chain *restful.FilterChain)

func (*Master) CreateWebService 

func (m *Master) CreateWebService()

func (*Master) GetClickModel 

func (m *Master) GetClickModel(version *protocol.VersionInfo, sender protocol.Master_GetClickModelServer) error

GetClickModel returns latest click model.

func (*Master) GetMeta 

func (m *Master) GetMeta(ctx context.Context, nodeInfo *protocol.NodeInfo) (*protocol.Meta, error)

GetMeta returns latest configuration.

func (*Master) GetRankingModel 

func (m *Master) GetRankingModel(version *protocol.VersionInfo, sender protocol.Master_GetRankingModelServer) error

GetRankingModel returns latest ranking model.

func (*Master) LoadDataFromDatabase 

func (m *Master) LoadDataFromDatabase(database data.Database, posFeedbackTypes, readTypes []string, itemTTL, positiveFeedbackTTL uint, evaluator *OnlineEvaluator) (
	rankingDataset *ranking.DataSet, clickDataset *click.Dataset, latestItems map[string][]cache.Scored, popularItems map[string][]cache.Scored, err error)

LoadDataFromDatabase loads dataset from data store.

func (*Master) PushTaskInfo 

func (m *Master) PushTaskInfo(
	_ context.Context,
	in *protocol.PushTaskInfoRequest) (*protocol.PushTaskInfoResponse, error)

func (*Master) RunPrivilegedTasksLoop 

func (m *Master) RunPrivilegedTasksLoop()

func (*Master) RunRagtagTasksLoop 

func (m *Master) RunRagtagTasksLoop()

RunRagtagTasksLoop searches optimal recommendation model in background. It never modifies variables other than rankingModelSearcher, clickSearchedModel and clickSearchedScore.

func (*Master) Serve 

func (m *Master) Serve()

Serve starts the master node.

func (*Master) StartHttpServer 

func (m *Master) StartHttpServer()

type Node 

type Node struct {
	Name          string
	Type          string
	IP            string
	HttpPort      int64
	BinaryVersion string
}

Node could be worker node for server node.

func NewNode 

func NewNode(ctx context.Context, nodeInfo *protocol.NodeInfo) *Node

NewNode creates a node from Context and NodeInfo.

type OnlineEvaluator 

type OnlineEvaluator struct {
	ReadFeedbacks      []map[int32]*i32set.Set
	PositiveFeedbacks  map[string][]lo.Tuple3[int32, int32, time.Time]
	ReverseIndex       map[lo.Tuple2[int32, int32]]time.Time
	EvaluateDays       int
	TruncatedDateToday time.Time
}

func NewOnlineEvaluator 

func NewOnlineEvaluator() *OnlineEvaluator

func (*OnlineEvaluator) Evaluate 

func (evaluator *OnlineEvaluator) Evaluate() []server.Measurement

func (*OnlineEvaluator) Positive 

func (evaluator *OnlineEvaluator) Positive(feedbackType string, userIndex, itemIndex int32, timestamp time.Time)

func (*OnlineEvaluator) Read 

func (evaluator *OnlineEvaluator) Read(userIndex, itemIndex int32, timestamp time.Time)

type SinglePageAppFileSystem 

type SinglePageAppFileSystem struct {
	// contains filtered or unexported fields
}

SinglePageAppFileSystem is the file system for single page app.

func (*SinglePageAppFileSystem) Open 

func (fs *SinglePageAppFileSystem) Open(name string) (http.File, error)

Open index.html if required file not exists.

type Status 

type Status struct {
	BinaryVersion           string
	NumServers              int
	NumWorkers              int
	NumUsers                int
	NumItems                int
	NumUserLabels           int
	NumItemLabels           int
	NumTotalPosFeedback     int
	NumValidPosFeedback     int
	NumValidNegFeedback     int
	PopularItemsUpdateTime  time.Time
	LatestItemsUpdateTime   time.Time
	MatchingModelFitTime    time.Time
	MatchingModelScore      ranking.Score
	RankingModelFitTime     time.Time
	RankingModelScore       click.Score
	UserNeighborIndexRecall float32
	ItemNeighborIndexRecall float32
	MatchingIndexRecall     float32
}

type User 

type User struct {
	data.User
	LastActiveTime time.Time
	LastUpdateTime time.Time
}

type UserIterator 

type UserIterator struct {
	Cursor string
	Users  []User
}

type Vectors 

type Vectors struct {
	// contains filtered or unexported fields
}

func NewVectors 

func NewVectors(connections, connected [][]int32, weights []float32) *Vectors

func (*Vectors) Distance 

func (v *Vectors) Distance(i, j int) float32

func (*Vectors) Neighbors 

func (v *Vectors) Neighbors(i int) []int32

type VectorsInterface 

type VectorsInterface interface {
	Distance(i, j int) float32
	Neighbors(i int) []int32
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.