stats

package
v0.0.0-...-418c65a Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 26, 2019 License: Apache-2.0 Imports: 16 Imported by: 0

Documentation

Overview

Package stats provides functions to generate and query pre-computed test and task statistics. The statistics are aggregated per day and a combination of (test, task, variant, distro, project, requester) for tests and a combination of (task, variant, distro, project, requester) for tasks. For tests intermediate hourly statistics are also stored to avoid some re-computation.

Index

Constants

View Source 
const (
	MaxQueryLimit             = 1001
	GroupByTest       GroupBy = "test"
	GroupByTask       GroupBy = "task"
	GroupByVariant    GroupBy = "variant"
	GroupByDistro     GroupBy = "distro"
	SortEarliestFirst Sort    = "earliest"
	SortLatestFirst   Sort    = "latest"
)
View Source 
const (
	DailyTaskStatsCollection = "daily_task_stats"
)

Variables

View Source 
var (
	DbTestStatsIdTaskNameKey     = bsonutil.MustHaveTag(DbTestStatsId{}, "TaskName")
	DbTestStatsIdBuildVariantKey = bsonutil.MustHaveTag(DbTestStatsId{}, "BuildVariant")
	DbTestStatsIdDistroKey       = bsonutil.MustHaveTag(DbTestStatsId{}, "Distro")

	// BSON dotted field names for test stats id elements
	DbTestStatsIdTestFileKeyFull     = bsonutil.GetDottedKeyName(dbTestStatsIdKey, dbTestStatsIdTestFileKey)
	DbTestStatsIdTaskNameKeyFull     = bsonutil.GetDottedKeyName(dbTestStatsIdKey, DbTestStatsIdTaskNameKey)
	DbTestStatsIdBuildVariantKeyFull = bsonutil.GetDottedKeyName(dbTestStatsIdKey, DbTestStatsIdBuildVariantKey)
	DbTestStatsIdDistroKeyFull       = bsonutil.GetDottedKeyName(dbTestStatsIdKey, DbTestStatsIdDistroKey)
	DbTestStatsIdProjectKeyFull      = bsonutil.GetDottedKeyName(dbTestStatsIdKey, dbTestStatsIdProjectKey)
	DbTestStatsIdRequesterKeyFull    = bsonutil.GetDottedKeyName(dbTestStatsIdKey, dbTestStatsIdRequesterKey)
	DbTestStatsIdDateKeyFull         = bsonutil.GetDottedKeyName(dbTestStatsIdKey, dbTestStatsIdDateKey)
)
View Source 
var (
	// BSON fields for the task stats id struct
	DbTaskStatsIdTaskNameKey     = bsonutil.MustHaveTag(DbTaskStatsId{}, "TaskName")
	DbTaskStatsIdBuildVariantKey = bsonutil.MustHaveTag(DbTaskStatsId{}, "BuildVariant")
	DbTaskStatsIdDistroKey       = bsonutil.MustHaveTag(DbTaskStatsId{}, "Distro")
	DbTaskStatsIdProjectKey      = bsonutil.MustHaveTag(DbTaskStatsId{}, "Project")
	DbTaskStatsIdRequesterKey    = bsonutil.MustHaveTag(DbTaskStatsId{}, "Requester")
	DbTaskStatsIdDateKey         = bsonutil.MustHaveTag(DbTaskStatsId{}, "Date")

	// BSON fields for the test stats struct
	DbTaskStatsIdKey                 = bsonutil.MustHaveTag(DbTaskStats{}, "Id")
	DbTaskStatsNumSuccessKey         = bsonutil.MustHaveTag(DbTaskStats{}, "NumSuccess")
	DbTaskStatsNumFailedKey          = bsonutil.MustHaveTag(DbTaskStats{}, "NumFailed")
	DbTaskStatsNumTestFailedKey      = bsonutil.MustHaveTag(DbTaskStats{}, "NumTestFailed")
	DbTaskStatsNumSetupFailedKey     = bsonutil.MustHaveTag(DbTaskStats{}, "NumSetupFailed")
	DbTaskStatsNumSystemFailedKey    = bsonutil.MustHaveTag(DbTaskStats{}, "NumSystemFailed")
	DbTaskStatsNumTimeoutKey         = bsonutil.MustHaveTag(DbTaskStats{}, "NumTimeout")
	DbTaskStatsAvgDurationSuccessKey = bsonutil.MustHaveTag(DbTaskStats{}, "AvgDurationSuccess")
	DbTaskStatsLastUpdateKey         = bsonutil.MustHaveTag(DbTaskStats{}, "LastUpdate")

	// BSON dotted field names for task stats id elements
	DbTaskStatsIdTaskNameKeyFull     = bsonutil.GetDottedKeyName(DbTaskStatsIdKey, DbTaskStatsIdTaskNameKey)
	DbTaskStatsIdBuildVariantKeyFull = bsonutil.GetDottedKeyName(DbTaskStatsIdKey, DbTaskStatsIdBuildVariantKey)
	DbTaskStatsIdDistroKeyFull       = bsonutil.GetDottedKeyName(DbTaskStatsIdKey, DbTaskStatsIdDistroKey)
	DbTaskStatsIdDateKeyFull         = bsonutil.GetDottedKeyName(DbTaskStatsIdKey, DbTaskStatsIdDateKey)
)
View Source 
var (
	// BSON fields for the test stats struct
	TestStatsTestFileKey        = bsonutil.MustHaveTag(TestStats{}, "TestFile")
	TestStatsTaskNameKey        = bsonutil.MustHaveTag(TestStats{}, "TaskName")
	TestStatsBuildVariantKey    = bsonutil.MustHaveTag(TestStats{}, "BuildVariant")
	TestStatsDistroKey          = bsonutil.MustHaveTag(TestStats{}, "Distro")
	TestStatsDateKey            = bsonutil.MustHaveTag(TestStats{}, "Date")
	TestStatsNumPassKey         = bsonutil.MustHaveTag(TestStats{}, "NumPass")
	TestStatsNumFailKey         = bsonutil.MustHaveTag(TestStats{}, "NumFail")
	TestStatsAvgDurationPassKey = bsonutil.MustHaveTag(TestStats{}, "AvgDurationPass")
	TestStatsLastUpdateKey      = bsonutil.MustHaveTag(TestStats{}, "LastUpdate")

	// BSON fields for the task stats struct
	TaskStatsTaskNameKey           = bsonutil.MustHaveTag(TaskStats{}, "TaskName")
	TaskStatsBuildVariantKey       = bsonutil.MustHaveTag(TaskStats{}, "BuildVariant")
	TaskStatsDistroKey             = bsonutil.MustHaveTag(TaskStats{}, "Distro")
	TaskStatsDateKey               = bsonutil.MustHaveTag(TaskStats{}, "Date")
	TaskStatsNumSuccessKey         = bsonutil.MustHaveTag(TaskStats{}, "NumSuccess")
	TaskStatsNumFailedKey          = bsonutil.MustHaveTag(TaskStats{}, "NumFailed")
	TaskStatsNumTotalKey           = bsonutil.MustHaveTag(TaskStats{}, "NumTotal")
	TaskStatsNumTestFailedKey      = bsonutil.MustHaveTag(TaskStats{}, "NumTestFailed")
	TaskStatsNumSetupFailedKey     = bsonutil.MustHaveTag(TaskStats{}, "NumSetupFailed")
	TaskStatsNumSystemFailedKey    = bsonutil.MustHaveTag(TaskStats{}, "NumSystemFailed")
	TaskStatsNumTimeoutKey         = bsonutil.MustHaveTag(TaskStats{}, "NumTimeout")
	TaskStatsAvgDurationSuccessKey = bsonutil.MustHaveTag(TaskStats{}, "AvgDurationSuccess")
	TaskStatsLastUpdateKey         = bsonutil.MustHaveTag(TaskStats{}, "LastUpdate")
)

Functions

func BuildMatchArrayExpression 

func BuildMatchArrayExpression(values []string) interface{}

BuildMatchArrayExpression builds an expression to match any of the values in the array argument.

func BuildPaginationOrBranches 

func BuildPaginationOrBranches(fields []PaginationField) []bson.M

BuildPaginationOrBranches builds and returns the $or branches of the pagination constraints. fields is an array of field names, they must be in the same order as the sort order. operators is a list of MongoDB comparison operators ("$gte", "$gt", "$lte", "$lt") for the fields. values is a list of values for the fields. func (filter StatsFilter) buildPaginationOrBranches(fields []PaginationField) []bson.M {

func GenerateDailyTaskStats 

func GenerateDailyTaskStats(ctx context.Context, opts GenerateOptions) error

GenerateDailyTaskStats aggregates the hourly task stats present in the database and saves the resulting daily task stats documents for the project, requester, day, and tasks specified. The day covered is the UTC day corresponding to the given `day` parameter.

func GenerateDailyTestStatsFromHourly 

func GenerateDailyTestStatsFromHourly(ctx context.Context, opts GenerateOptions) error

GenerateDailyTestStatsFromHourly aggregates the hourly test stats present in the database and saves the resulting daily test stats documents for the project, requester, day, and tasks specified. The day covered is the UTC day corresponding to the given `day` parameter.

func GenerateHourlyTestStats 

func GenerateHourlyTestStats(ctx context.Context, opts GenerateOptions) error

GenerateHourlyTestStats aggregates task and testresults prsent in the database and saves the resulting hourly test stats documents for the project, requester, hour, and tasks specified. The hour covered is the UTC hour corresponding to the given `hour` parameter.

func GetDailyTestDoc 

func GetDailyTestDoc(id DbTestStatsId) (*dbTestStats, error)

func GetHourlyTestDoc 

func GetHourlyTestDoc(id DbTestStatsId) (*dbTestStats, error)

func SortDateOrder 

func SortDateOrder(sort Sort) int

SortDateOrder returns the sort order specification (1, -1) for the date field corresponding to the Sort value.

func UpdateStatsStatus 

func UpdateStatsStatus(projectId string, lastJobRun time.Time, processedTasksUntil time.Time, runtime time.Duration) error

UpdateStatsStatus updates the status of the stats pre-computations for a project.

Types

type Array 

type Array []interface{}

Convenient type to use for arrays in pipeline definitions.

type DbTaskStats 

type DbTaskStats struct {
	Id                 DbTaskStatsId `bson:"_id"`
	NumSuccess         int           `bson:"num_success"`
	NumFailed          int           `bson:"num_failed"`
	NumTimeout         int           `bson:"num_timeout"`
	NumTestFailed      int           `bson:"num_test_failed"`
	NumSystemFailed    int           `bson:"num_system_failed"`
	NumSetupFailed     int           `bson:"num_setup_failed"`
	AvgDurationSuccess float64       `bson:"avg_duration_success"`
	LastUpdate         time.Time     `bson:"last_update"`
}

DbTaskStats represents the daily_task_stats documents.

func GetDailyTaskDoc 

func GetDailyTaskDoc(id DbTaskStatsId) (*DbTaskStats, error)

func (*DbTaskStats) MarshalBSON 

func (d *DbTaskStats) MarshalBSON() ([]byte, error)

func (*DbTaskStats) UnmarshalBSON 

func (d *DbTaskStats) UnmarshalBSON(in []byte) error

type DbTaskStatsId 

type DbTaskStatsId struct {
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Project      string    `bson:"project"`
	Requester    string    `bson:"requester"`
	Date         time.Time `bson:"date"`
}

DbTaskStatsId represents the _id field for daily_task_stats documents.

type DbTestStatsId 

type DbTestStatsId struct {
	TestFile     string    `bson:"test_file"`
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Project      string    `bson:"project"`
	Requester    string    `bson:"requester"`
	Date         time.Time `bson:"date"`
}

DbTestStatsId represents the _id field for hourly_test_stats and daily_test_stats documents.

type FindStatsOptions 

type FindStatsOptions struct {
	ProjectID       string
	Requesters      []string
	Start           time.Time
	End             time.Time
	DisableOldTasks bool
}

type GenerateOptions 

type GenerateOptions struct {
	ProjectID       string
	Requester       string
	Tasks           []string
	Window          time.Time
	Runtime         time.Time
	DisableOldTasks bool
}

type GroupBy 

type GroupBy string

type PaginationField 

type PaginationField struct {
	Field      string
	Descending bool
	Strict     bool
	Value      interface{}
	NextValue  interface{}
}

PaginationField represents a statistics document field that is used to determine where to resume during pagination.

func (PaginationField) GetEqExpression 

func (pf PaginationField) GetEqExpression() interface{}

GetEqExpression returns an expression that can be used to match the documents which have the same field value or are in the same range as this PaginationField.

func (PaginationField) GetNextExpression 

func (pf PaginationField) GetNextExpression() bson.M

GetNextExpression returns an expression that can be used to match the documents which have a field value greater or smaller than the this PaginationField.

type Sort 

type Sort string

type StartAt 

type StartAt struct {
	Date         time.Time
	BuildVariant string
	Task         string
	Test         string
	Distro       string
}

StartAt represents parameters that allow a search query to resume at a specific point. Used for pagination.

func StartAtFromTaskStats 

func StartAtFromTaskStats(taskStats *TaskStats) StartAt

StartAtFromTaskStats creates a StartAt that can be used to resume a task stats query. Using the returned StartAt the given TaskStats will be the first result.

func StartAtFromTestStats 

func StartAtFromTestStats(testStats *TestStats) StartAt

StartAtFromTestStats creates a StartAt that can be used to resume a test stats query. Using the returned StartAt the given TestStats will be the first result.

type StatsFilter 

type StatsFilter struct {
	Project    string
	Requesters []string
	AfterDate  time.Time
	BeforeDate time.Time

	Tests         []string
	Tasks         []string
	BuildVariants []string
	Distros       []string

	GroupNumDays int
	GroupBy      GroupBy
	StartAt      *StartAt
	Limit        int
	Sort         Sort
}

StatsFilter represents search and aggregation parameters when querying the test or task statistics.

func (StatsFilter) BuildTaskStatsQueryGroupStage 

func (filter StatsFilter) BuildTaskStatsQueryGroupStage() bson.M

TaskStatsQueryPipeline creates an aggregation pipeline to query task statistics.

func (StatsFilter) BuildTaskStatsQueryProjectStage 

func (filter StatsFilter) BuildTaskStatsQueryProjectStage() bson.M

buildTaskStatsQueryProjectStage creates an aggregation project stage to query task statistics.

func (StatsFilter) BuildTaskStatsQuerySortStage 

func (filter StatsFilter) BuildTaskStatsQuerySortStage() bson.M

buildTaskStatsQuerySortStage creates an aggregation sort stage to query task statistics.

func (StatsFilter) TaskStatsQueryPipeline 

func (filter StatsFilter) TaskStatsQueryPipeline() []bson.M

TaskStatsQueryPipeline creates an aggregation pipeline to query task statistics.

func (*StatsFilter) ValidateCommon 

func (f *StatsFilter) ValidateCommon() error

validateCommon performs common validations regardless of the filter's intended use.

func (*StatsFilter) ValidateForTasks 

func (f *StatsFilter) ValidateForTasks() error

ValidateForTasks use with test stats validates that the StatsFilter struct is valid for use with task stats.

func (*StatsFilter) ValidateForTests 

func (f *StatsFilter) ValidateForTests() error

ValidateForTests validates that the StatsFilter struct is valid for use with test stats.

type StatsStatus 

type StatsStatus struct {
	ProjectId           string        `bson:"_id"`
	LastJobRun          time.Time     `bson:"last_job_run"`
	ProcessedTasksUntil time.Time     `bson:"processed_tasks_until"`
	Runtime             time.Duration `bson:"runtime"`
}

StatsStatus represents the status for stats pre-computations for a project. LastJobRun is the start date of the last successful pre-computation job that ran for the project. ProcessedTasksUntil is the date before which all finished tasks have been processed. It is usually the same as LastJobRun unless a previous job has failed and the stats computation has not caught up yet.

func GetStatsStatus 

func GetStatsStatus(projectId string) (StatsStatus, error)

GetStatsStatus retrieves the status of the stats pre-computations for a project.

type StatsToUpdate 

type StatsToUpdate struct {
	ProjectId string    `bson:"project"`
	Requester string    `bson:"requester"`
	Hour      time.Time `bson:"date"`
	Day       time.Time `bson:"day"`
	Tasks     []string  `bson:"task_names"`
}

func FindStatsToUpdate 

func FindStatsToUpdate(opts FindStatsOptions) ([]StatsToUpdate, error)

FidnStatsToUpdate finds the stats that need to be updated as a result of tasks finishing between 'start' and 'end'. The results are ordered by project id, then hour, then requester.

type TaskStats 

type TaskStats struct {
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Date         time.Time `bson:"date"`

	NumTotal           int       `bson:"num_total"`
	NumSuccess         int       `bson:"num_success"`
	NumFailed          int       `bson:"num_failed"`
	NumTimeout         int       `bson:"num_timeout"`
	NumTestFailed      int       `bson:"num_test_failed"`
	NumSystemFailed    int       `bson:"num_system_failed"`
	NumSetupFailed     int       `bson:"num_setup_failed"`
	AvgDurationSuccess float64   `bson:"avg_duration_success"`
	LastUpdate         time.Time `bson:"last_update"`
}

TaskStats represents task execution statistics.

func GetTaskStats 

func GetTaskStats(filter StatsFilter) ([]TaskStats, error)

GetTaskStats queries the precomputed task statistics using a filter.

func (*TaskStats) MarshalBSON 

func (s *TaskStats) MarshalBSON() ([]byte, error)

func (*TaskStats) UnmarshalBSON 

func (s *TaskStats) UnmarshalBSON(in []byte) error

type TestStats 

type TestStats struct {
	TestFile     string    `bson:"test_file"`
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Date         time.Time `bson:"date"`

	NumPass         int       `bson:"num_pass"`
	NumFail         int       `bson:"num_fail"`
	AvgDurationPass float64   `bson:"avg_duration_pass"`
	LastUpdate      time.Time `bson:"last_update"`
}

TestStats represents test execution statistics.

func GetTestStats 

func GetTestStats(filter StatsFilter) ([]TestStats, error)

GetTestStats queries the precomputed test statistics using a filter.

func (*TestStats) MarshalBSON 

func (s *TestStats) MarshalBSON() ([]byte, error)

func (*TestStats) UnmarshalBSON 

func (s *TestStats) UnmarshalBSON(in []byte) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.