master

package
v0.3.2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 1, 2022 License: Apache-2.0 Imports: 46 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	ServerNode = "Server"
	WorkerNode = "Worker"
)
View Source 
const (
	ClickThroughRate = "ClickThroughRate"

	TaskLoadDataset        = "Load dataset"
	TaskFindItemNeighbors  = "Find neighbors of items"
	TaskFindUserNeighbors  = "Find neighbors of users"
	TaskAnalyze            = "Analyze click-through rate"
	TaskFitRankingModel    = "Fit collaborative filtering model"
	TaskFitClickModel      = "Fit click-through rate prediction model"
	TaskSearchRankingModel = "Search collaborative filtering  model"
	TaskSearchClickModel   = "Search click-through rate prediction model"
)

Variables

View Source 
var (
	GetRankingModelSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "get_ranking_model_seconds",
	})
	GetClickModelSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "get_click_model_seconds",
	})
	GetUserIndexSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "get_user_index_seconds",
	})
	FindUserNeighborsSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_user_neighbors_seconds",
	})
	FindItemNeighborsSeconds = promauto.NewHistogram(prometheus.HistogramOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "find_item_neighbors_seconds",
	})

	MatchingTop10NDCG = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "matching_model_ndcg_at_10",
	})
	MatchingTop10Precision = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "matching_model_precision_at_10",
	})
	MatchingTop10Recall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "matching_model_recall_at_10",
	})
	RankingPrecision = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_precision",
	})
	RankingRecall = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_recall",
	})
	RankingAUC = promauto.NewGauge(prometheus.GaugeOpts{
		Namespace: "gorse",
		Subsystem: "master",
		Name:      "ranking_model_auc",
	})
)

Functions

This section is empty.

Types

type Feedback 

type Feedback struct {
	FeedbackType string
	UserId       string
	Item         data.Item
	Timestamp    time.Time
	Comment      string
}

type LocalCache 

type LocalCache struct {
	RankingModelName    string
	RankingModelVersion int64
	RankingModel        ranking.Model
	RankingModelScore   ranking.Score
	UserIndexVersion    int64
	UserIndex           base.Index
	ClickModelVersion   int64
	ClickModelScore     click.Score
	ClickModel          click.FactorizationMachine
	// contains filtered or unexported fields
}

LocalCache is local cache for the master node.

func LoadLocalCache 

func LoadLocalCache(path string) (*LocalCache, error)

LoadLocalCache loads local cache from a file. If the ranking model is invalid, RankingModel == nil. If the click model is invalid, ClickModel == nil.

func (*LocalCache) WriteLocalCache 

func (c *LocalCache) WriteLocalCache() error

WriteLocalCache writes local cache to a file.

type Master 

type Master struct {
	protocol.UnimplementedMasterServer
	server.RestServer
	// contains filtered or unexported fields
}

Master is the master node.

func NewMaster 

func NewMaster(cfg *config.Config, cacheFile string) *Master

NewMaster creates a master node.

func (*Master) CreateWebService 

func (m *Master) CreateWebService()

func (*Master) FinishTask added in v0.2.4 

func (m *Master) FinishTask(
	_ context.Context,
	in *protocol.FinishTaskRequest) (*protocol.FinishTaskResponse, error)

func (*Master) GetClickModel added in v0.2.2 

func (m *Master) GetClickModel(version *protocol.VersionInfo, sender protocol.Master_GetClickModelServer) error

GetClickModel returns latest click model.

func (*Master) GetMeta 

func (m *Master) GetMeta(ctx context.Context, nodeInfo *protocol.NodeInfo) (*protocol.Meta, error)

GetMeta returns latest configuration.

func (*Master) GetRankingModel 

func (m *Master) GetRankingModel(version *protocol.VersionInfo, sender protocol.Master_GetRankingModelServer) error

GetRankingModel returns latest ranking model.

func (*Master) GetUserIndex 

func (m *Master) GetUserIndex(version *protocol.VersionInfo, sender protocol.Master_GetUserIndexServer) error

GetUserIndex returns latest user index.

func (*Master) LoadDataFromDatabase added in v0.2.6 

func (m *Master) LoadDataFromDatabase(database data.Database, posFeedbackTypes, readTypes []string, itemTTL, positiveFeedbackTTL uint) (
	rankingDataset *ranking.DataSet, clickDataset *click.Dataset, latestItems map[string][]cache.Scored, popularItems map[string][]cache.Scored, err error)

LoadDataFromDatabase loads dataset from data store.

func (*Master) RunPrivilegedTasksLoop added in v0.2.4 

func (m *Master) RunPrivilegedTasksLoop()

func (*Master) RunRagtagTasksLoop added in v0.2.4 

func (m *Master) RunRagtagTasksLoop()

RunRagtagTasksLoop searches optimal recommendation model in background. It never modifies variables other than rankingModelSearcher, clickSearchedModel and clickSearchedScore.

func (*Master) Serve 

func (m *Master) Serve()

Serve starts the master node.

func (*Master) StartHttpServer 

func (m *Master) StartHttpServer()

func (*Master) StartTask added in v0.2.4 

func (m *Master) StartTask(
	_ context.Context,
	in *protocol.StartTaskRequest) (*protocol.StartTaskResponse, error)

func (*Master) UpdateTask added in v0.2.4 

func (m *Master) UpdateTask(
	_ context.Context,
	in *protocol.UpdateTaskRequest) (*protocol.UpdateTaskResponse, error)

type Node 

type Node struct {
	Name     string
	Type     string
	IP       string
	HttpPort int64
}

Node could be worker node for server node.

func NewNode 

func NewNode(ctx context.Context, nodeInfo *protocol.NodeInfo) *Node

NewNode creates a node from Context and NodeInfo.

type SinglePageAppFileSystem 

type SinglePageAppFileSystem struct {
	// contains filtered or unexported fields
}

SinglePageAppFileSystem is the file system for single page app.

func (*SinglePageAppFileSystem) Open 

func (fs *SinglePageAppFileSystem) Open(name string) (http.File, error)

Open index.html if required file not exists.

type Status 

type Status struct {
	NumServers              int
	NumWorkers              int
	NumUsers                int
	NumItems                int
	NumUserLabels           int
	NumItemLabels           int
	NumTotalPosFeedback     int
	NumValidPosFeedback     int
	NumValidNegFeedback     int
	PopularItemsUpdateTime  time.Time
	LatestItemsUpdateTime   time.Time
	MatchingModelFitTime    time.Time
	MatchingModelScore      ranking.Score
	RankingModelFitTime     time.Time
	RankingModelScore       click.Score
	UserNeighborIndexRecall float32
	ItemNeighborIndexRecall float32
	MatchingIndexRecall     float32
}

type SubTaskTracker added in v0.2.4 

type SubTaskTracker struct {
	*TaskTracker
	Offset int
	Total  int
}

SubTaskTracker tracks part of progress of a task.

func (*SubTaskTracker) Fail added in v0.3.2 

func (tt *SubTaskTracker) Fail(err string)

Fail reports the error message.

func (*SubTaskTracker) Finish added in v0.2.4 

func (tt *SubTaskTracker) Finish()

Finish a task.

func (*SubTaskTracker) Start added in v0.2.4 

func (tt *SubTaskTracker) Start(total int)

Start a task.

func (*SubTaskTracker) SubTracker added in v0.2.4 

func (tt *SubTaskTracker) SubTracker() model.Tracker

SubTracker creates a sub tracker of a sub tracker.

func (*SubTaskTracker) Suspend added in v0.2.4 

func (tt *SubTaskTracker) Suspend(flag bool)

func (*SubTaskTracker) Update added in v0.2.4 

func (tt *SubTaskTracker) Update(done int)

Update the progress of current task.

type Task added in v0.2.4 

type Task struct {
	Name       string
	Status     TaskStatus
	Done       int
	Total      int
	StartTime  time.Time
	FinishTime time.Time
	Error      string
}

Task progress information.

type TaskMonitor added in v0.2.4 

type TaskMonitor struct {
	TaskLock sync.Mutex
	Tasks    map[string]*Task
}

TaskMonitor monitors the progress of all tasks.

func NewTaskMonitor added in v0.2.4 

func NewTaskMonitor() *TaskMonitor

NewTaskMonitor creates a TaskMonitor and add pending tasks.

func (*TaskMonitor) Fail added in v0.2.5 

func (tm *TaskMonitor) Fail(name, err string)

func (*TaskMonitor) Finish added in v0.2.4 

func (tm *TaskMonitor) Finish(name string)

Finish a task.

func (*TaskMonitor) Get added in v0.2.4 

func (tm *TaskMonitor) Get(name string) int

Get the progress of a task.

func (*TaskMonitor) List added in v0.2.4 

func (tm *TaskMonitor) List() []Task

List all tasks.

func (*TaskMonitor) NewTaskTracker added in v0.2.4 

func (tm *TaskMonitor) NewTaskTracker(name string) *TaskTracker

NewTaskTracker creates a TaskTracker from TaskMonitor.

func (*TaskMonitor) Pending added in v0.2.4 

func (tm *TaskMonitor) Pending(name string)

Pending a task.

func (*TaskMonitor) Start added in v0.2.4 

func (tm *TaskMonitor) Start(name string, total int)

Start a task.

func (*TaskMonitor) Suspend added in v0.2.4 

func (tm *TaskMonitor) Suspend(name string, flag bool)

Suspend a task.

func (*TaskMonitor) Update added in v0.2.4 

func (tm *TaskMonitor) Update(name string, done int)

Update the progress of a task.

type TaskRunner added in v0.2.4 

type TaskRunner struct {
	*TaskScheduler
	Name string
}

TaskRunner is a TaskScheduler bounded with a task.

func (*TaskRunner) Lock added in v0.2.4 

func (locker *TaskRunner) Lock()

Lock gets the permission to run task.

func (*TaskRunner) UnLock added in v0.2.4 

func (locker *TaskRunner) UnLock()

UnLock returns the permission to run task.

type TaskScheduler added in v0.2.4 

type TaskScheduler struct {
	*sync.Cond
	Privileged *strset.Set
	Running    bool
}

TaskScheduler schedules that pre-locked tasks are executed first.

func NewTaskScheduler added in v0.2.4 

func NewTaskScheduler() *TaskScheduler

NewTaskScheduler creates a TaskScheduler.

func (*TaskScheduler) Lock added in v0.2.4 

func (t *TaskScheduler) Lock(name string)

Lock gets the permission to run task.

func (*TaskScheduler) NewRunner added in v0.2.4 

func (t *TaskScheduler) NewRunner(name string) *TaskRunner

NewRunner

func (*TaskScheduler) PreLock added in v0.2.4 

func (t *TaskScheduler) PreLock(name string)

PreLock a task, the task has the privilege to run first than un-pre-clocked tasks.

func (*TaskScheduler) UnLock added in v0.2.4 

func (t *TaskScheduler) UnLock(name string)

UnLock returns the permission to run task.

type TaskStatus added in v0.2.4 

type TaskStatus string
const (
	TaskStatusPending   TaskStatus = "Pending"
	TaskStatusComplete  TaskStatus = "Complete"
	TaskStatusRunning   TaskStatus = "Running"
	TaskStatusSuspended TaskStatus = "Suspended"
	TaskStatusFailed    TaskStatus = "Failed"
)

type TaskTracker added in v0.2.4 

type TaskTracker struct {
	// contains filtered or unexported fields
}

TaskTracker tracks the progress of a task.

func (*TaskTracker) Fail added in v0.3.2 

func (tt *TaskTracker) Fail(err string)

func (*TaskTracker) Finish added in v0.2.4 

func (tt *TaskTracker) Finish()

Finish the task.

func (*TaskTracker) Start added in v0.2.4 

func (tt *TaskTracker) Start(total int)

Start the task.

func (*TaskTracker) SubTracker added in v0.2.4 

func (tt *TaskTracker) SubTracker() model.Tracker

SubTracker creates a sub tracker

func (*TaskTracker) Suspend added in v0.2.4 

func (tt *TaskTracker) Suspend(flag bool)

func (*TaskTracker) Update added in v0.2.4 

func (tt *TaskTracker) Update(done int)

Update the progress of this task.

type Tasks added in v0.2.4 

type Tasks []Task

Tasks is used to sort []Task.

func (Tasks) Len added in v0.2.4 

func (t Tasks) Len() int

Len is used to sort []Task.

func (Tasks) Less added in v0.2.4 

func (t Tasks) Less(i, j int) bool

Less is used to sort []Task.

func (Tasks) Swap added in v0.2.4 

func (t Tasks) Swap(i, j int)

Swap is used to sort []Task.

type User 

type User struct {
	data.User
	LastActiveTime time.Time
	LastUpdateTime time.Time
}

type UserIterator 

type UserIterator struct {
	Cursor string
	Users  []User
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.