taskstats

package
v0.0.0-...-be7f909 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 22, 2024 License: Apache-2.0 Imports: 18 Imported by: 0

Documentation

Overview

Package taskstats provides functions to generate and query pre-computed and task statistics. The statistics are aggregated per day and a combination of (project, variant, distro, task, requester).

Index

Constants

View Source 
const (
	DailyTaskStatsCollection   = "daily_task_stats"
	DailyStatsStatusCollection = "daily_stats_status"
)
View Source 
const (
	MaxQueryLimit             = 1001
	GroupByTask       GroupBy = "task"
	GroupByVariant    GroupBy = "variant"
	GroupByDistro     GroupBy = "distro"
	SortEarliestFirst Sort    = "earliest"
	SortLatestFirst   Sort    = "latest"
)

Variables

View Source 
var (
	// BSON fields for the task stats ID struct.
	DBTaskStatsIDTaskNameKey     = bsonutil.MustHaveTag(DBTaskStatsID{}, "TaskName")
	DBTaskStatsIDBuildVariantKey = bsonutil.MustHaveTag(DBTaskStatsID{}, "BuildVariant")
	DBTaskStatsIDDistroKey       = bsonutil.MustHaveTag(DBTaskStatsID{}, "Distro")
	DBTaskStatsIDProjectKey      = bsonutil.MustHaveTag(DBTaskStatsID{}, "Project")
	DBTaskStatsIDRequesterKey    = bsonutil.MustHaveTag(DBTaskStatsID{}, "Requester")
	DBTaskStatsIDDateKey         = bsonutil.MustHaveTag(DBTaskStatsID{}, "Date")

	// BSON fields for the task stats struct.
	DBTaskStatsIDKey                 = bsonutil.MustHaveTag(DBTaskStats{}, "Id")
	DBTaskStatsNumSuccessKey         = bsonutil.MustHaveTag(DBTaskStats{}, "NumSuccess")
	DBTaskStatsNumFailedKey          = bsonutil.MustHaveTag(DBTaskStats{}, "NumFailed")
	DBTaskStatsNumTestFailedKey      = bsonutil.MustHaveTag(DBTaskStats{}, "NumTestFailed")
	DBTaskStatsNumSetupFailedKey     = bsonutil.MustHaveTag(DBTaskStats{}, "NumSetupFailed")
	DBTaskStatsNumSystemFailedKey    = bsonutil.MustHaveTag(DBTaskStats{}, "NumSystemFailed")
	DBTaskStatsNumTimeoutKey         = bsonutil.MustHaveTag(DBTaskStats{}, "NumTimeout")
	DBTaskStatsAvgDurationSuccessKey = bsonutil.MustHaveTag(DBTaskStats{}, "AvgDurationSuccess")
	DBTaskStatsLastUpdateKey         = bsonutil.MustHaveTag(DBTaskStats{}, "LastUpdate")

	// BSON dotted field names for task stats ID elements.
	DBTaskStatsIDTaskNameKeyFull     = bsonutil.GetDottedKeyName(DBTaskStatsIDKey, DBTaskStatsIDTaskNameKey)
	DBTaskStatsIDBuildVariantKeyFull = bsonutil.GetDottedKeyName(DBTaskStatsIDKey, DBTaskStatsIDBuildVariantKey)
	DBTaskStatsIDDistroKeyFull       = bsonutil.GetDottedKeyName(DBTaskStatsIDKey, DBTaskStatsIDDistroKey)
	DBTaskStatsIDProjectKeyFull      = bsonutil.GetDottedKeyName(DBTaskStatsIDKey, DBTaskStatsIDProjectKey)
	DBTaskStatsIDRequesterKeyFull    = bsonutil.GetDottedKeyName(DBTaskStatsIDKey, DBTaskStatsIDRequesterKey)
	DBTaskStatsIDDateKeyFull         = bsonutil.GetDottedKeyName(DBTaskStatsIDKey, DBTaskStatsIDDateKey)
)
View Source 
var (
	// BSON fields for the task stats struct
	TaskStatsTaskNameKey           = bsonutil.MustHaveTag(TaskStats{}, "TaskName")
	TaskStatsBuildVariantKey       = bsonutil.MustHaveTag(TaskStats{}, "BuildVariant")
	TaskStatsDistroKey             = bsonutil.MustHaveTag(TaskStats{}, "Distro")
	TaskStatsDateKey               = bsonutil.MustHaveTag(TaskStats{}, "Date")
	TaskStatsNumSuccessKey         = bsonutil.MustHaveTag(TaskStats{}, "NumSuccess")
	TaskStatsNumFailedKey          = bsonutil.MustHaveTag(TaskStats{}, "NumFailed")
	TaskStatsNumTotalKey           = bsonutil.MustHaveTag(TaskStats{}, "NumTotal")
	TaskStatsNumTestFailedKey      = bsonutil.MustHaveTag(TaskStats{}, "NumTestFailed")
	TaskStatsNumSetupFailedKey     = bsonutil.MustHaveTag(TaskStats{}, "NumSetupFailed")
	TaskStatsNumSystemFailedKey    = bsonutil.MustHaveTag(TaskStats{}, "NumSystemFailed")
	TaskStatsNumTimeoutKey         = bsonutil.MustHaveTag(TaskStats{}, "NumTimeout")
	TaskStatsAvgDurationSuccessKey = bsonutil.MustHaveTag(TaskStats{}, "AvgDurationSuccess")
	TaskStatsLastUpdateKey         = bsonutil.MustHaveTag(TaskStats{}, "LastUpdate")
)

Functions

func BuildMatchArrayExpression 

func BuildMatchArrayExpression(values []string) interface{}

BuildMatchArrayExpression builds an expression to match any of the values in the array argument.

func BuildPaginationOrBranches 

func BuildPaginationOrBranches(fields []PaginationField) []bson.M

BuildPaginationOrBranches builds and returns the $or branches of the pagination constraints. fields is an array of field names, they must be in the same order as the sort order. operators is a list of MongoDB comparison operators ("$gte", "$gt", "$lte", "$lt") for the fields. values is a list of values for the fields. func (filter StatsFilter) buildPaginationOrBranches(fields []PaginationField) []bson.M {

func GenerateStats 

func GenerateStats(ctx context.Context, opts GenerateStatsOptions) error

GenerateStats aggregates the tasks in the database into task stats documents for the given project, requester, day, and tasks specified. The day covered is the UTC day corresponding to the given day parameter.

func SortDateOrder 

func SortDateOrder(sort Sort) int

SortDateOrder returns the sort order specification (1, -1) for the date field corresponding to the Sort value.

func UpdateStatsStatus 

func UpdateStatsStatus(projectID string, lastJobRun, processedTasksUntil time.Time, runtime time.Duration) error

UpdateStatsStatus updates the status of the stats pre-computations for a project.

Types

type Array 

type Array []interface{}

Convenient type to use for arrays in pipeline definitions.

type DBTaskStats 

type DBTaskStats struct {
	Id                 DBTaskStatsID `bson:"_id"`
	NumSuccess         int           `bson:"num_success"`
	NumFailed          int           `bson:"num_failed"`
	NumTimeout         int           `bson:"num_timeout"`
	NumTestFailed      int           `bson:"num_test_failed"`
	NumSystemFailed    int           `bson:"num_system_failed"`
	NumSetupFailed     int           `bson:"num_setup_failed"`
	AvgDurationSuccess float64       `bson:"avg_duration_success"`
	LastUpdate         time.Time     `bson:"last_update"`
}

DBTaskStats represents the daily_task_stats documents.

func GetDailyTaskDoc 

func GetDailyTaskDoc(id DBTaskStatsID) (*DBTaskStats, error)

func (*DBTaskStats) MarshalBSON 

func (d *DBTaskStats) MarshalBSON() ([]byte, error)

func (*DBTaskStats) UnmarshalBSON 

func (d *DBTaskStats) UnmarshalBSON(in []byte) error

type DBTaskStatsID 

type DBTaskStatsID struct {
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Project      string    `bson:"project"`
	Requester    string    `bson:"requester"`
	Date         time.Time `bson:"date"`
}

DBTaskStatsID represents the _id field for daily_task_stats documents.

type FindStatsToUpdateOptions 

type FindStatsToUpdateOptions struct {
	ProjectID  string
	Requesters []string
	Start      time.Time
	End        time.Time
}

type GenerateStatsOptions 

type GenerateStatsOptions struct {
	ProjectID string
	Requester string
	Tasks     []string
	Date      time.Time
}

type GroupBy 

type GroupBy string

type PaginationField 

type PaginationField struct {
	Field      string
	Descending bool
	Strict     bool
	Value      interface{}
	NextValue  interface{}
}

PaginationField represents a statistics document field that is used to determine where to resume during pagination.

func (PaginationField) GetEqExpression 

func (pf PaginationField) GetEqExpression() interface{}

GetEqExpression returns an expression that can be used to match the documents which have the same field value or are in the same range as this PaginationField.

func (PaginationField) GetNextExpression 

func (pf PaginationField) GetNextExpression() bson.M

GetNextExpression returns an expression that can be used to match the documents which have a field value greater or smaller than the this PaginationField.

type Sort 

type Sort string

type StartAt 

type StartAt struct {
	Date         time.Time
	BuildVariant string
	Task         string
	Distro       string
}

StartAt represents parameters that allow a search query to resume at a specific point. Used for pagination.

type StatsFilter 

type StatsFilter struct {
	Project    string
	Requesters []string
	AfterDate  time.Time
	BeforeDate time.Time

	Tasks         []string
	BuildVariants []string
	Distros       []string

	GroupNumDays int
	GroupBy      GroupBy
	StartAt      *StartAt
	Limit        int
	Sort         Sort
}

StatsFilter represents search and aggregation parameters when querying the task statistics.

func (StatsFilter) BuildTaskStatsQueryGroupStage 

func (filter StatsFilter) BuildTaskStatsQueryGroupStage() bson.M

TaskStatsQueryPipeline creates an aggregation pipeline to query task statistics.

func (StatsFilter) BuildTaskStatsQueryProjectStage 

func (filter StatsFilter) BuildTaskStatsQueryProjectStage() bson.M

buildTaskStatsQueryProjectStage creates an aggregation project stage to query task statistics.

func (StatsFilter) BuildTaskStatsQuerySortStage 

func (filter StatsFilter) BuildTaskStatsQuerySortStage() bson.M

buildTaskStatsQuerySortStage creates an aggregation sort stage to query task statistics.

func (StatsFilter) TaskStatsQueryPipeline 

func (filter StatsFilter) TaskStatsQueryPipeline() []bson.M

TaskStatsQueryPipeline creates an aggregation pipeline to query task statistics.

func (*StatsFilter) ValidateCommon 

func (f *StatsFilter) ValidateCommon() error

validateCommon performs common validations regardless of the filter's intended use.

func (*StatsFilter) ValidateForTasks 

func (f *StatsFilter) ValidateForTasks() error

ValidateForTasks validates that the StatsFilter struct is valid for use with task stats.

type StatsStatus 

type StatsStatus struct {
	ProjectID string `bson:"_id"`
	// LastJobRun is the start date of the last successful pre-computation
	// job that ran for the project.
	LastJobRun time.Time `bson:"last_job_run"`
	// ProcessedTasksUntil is the date before which all finished tasks have
	// been processed. It is usually the same as LastJobRun unless a
	// previous job has failed and the stats computation has not caught up
	// yet.
	ProcessedTasksUntil time.Time `bson:"processed_tasks_until"`
	// Runtime is the amount of time the last successful pre-computation
	// job that for the project took to complete.
	Runtime time.Duration `bson:"runtime"`
}

StatsStatus represents the status for stats pre-computations for a project.

func GetStatsStatus 

func GetStatsStatus(projectID string) (StatsStatus, error)

GetStatsStatus retrieves the status of the stats pre-computations for a project.

type StatsToUpdate 

type StatsToUpdate struct {
	Requester string    `bson:"requester"`
	Day       time.Time `bson:"day"`
	Tasks     []string  `bson:"task_names"`
}

func FindStatsToUpdate 

func FindStatsToUpdate(opts FindStatsToUpdateOptions) ([]StatsToUpdate, error)

FindStatsToUpdate finds the stats that need to be updated as a result of tasks finishing between the given start and end times. The results are ordered are ordered by first by date, then requester.

type TaskStats 

type TaskStats struct {
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Date         time.Time `bson:"date"`

	NumTotal           int       `bson:"num_total"`
	NumSuccess         int       `bson:"num_success"`
	NumFailed          int       `bson:"num_failed"`
	NumTimeout         int       `bson:"num_timeout"`
	NumTestFailed      int       `bson:"num_test_failed"`
	NumSystemFailed    int       `bson:"num_system_failed"`
	NumSetupFailed     int       `bson:"num_setup_failed"`
	AvgDurationSuccess float64   `bson:"avg_duration_success"`
	LastUpdate         time.Time `bson:"last_update"`
}

TaskStats represents task execution statistics.

func GetTaskStats 

func GetTaskStats(filter StatsFilter) ([]TaskStats, error)

GetTaskStats queries the precomputed task statistics using a filter.

func (*TaskStats) MarshalBSON 

func (s *TaskStats) MarshalBSON() ([]byte, error)

func (*TaskStats) UnmarshalBSON 

func (s *TaskStats) UnmarshalBSON(in []byte) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.