stats

Documentation

Overview

Package stats provides functions to generate and query pre-computed test and task statistics. The statistics are aggregated per day and a combination of (test, task, variant, distro, project, requester) for tests and a combination of (task, variant, distro, project, requester) for tasks. For tests intermediate hourly statistics are also stored to avoid some re-computation.

Index

• Constants
• Variables
• func GenerateDailyTaskStats(projectId string, requester string, day time.Time, tasks []string, ...) error
• func GenerateDailyTestStatsFromHourly(projectId string, requester string, day time.Time, tasks []string, ...) error
• func GenerateHourlyTestStats(projectId string, requester string, hour time.Time, tasks []string, ...) error
• func GetDailyTaskDoc(id DbTaskStatsId) (*dbTaskStats, error)
• func GetDailyTestDoc(id DbTestStatsId) (*dbTestStats, error)
• func GetHourlyTestDoc(id DbTestStatsId) (*dbTestStats, error)
• func UpdateStatsStatus(projectId string, lastJobRun time.Time, processedTasksUntil time.Time) error
• type DbTaskStatsId
• type DbTestStatsId
• type GroupBy
• type Sort
• type StartAt
  • func StartAtFromTaskStats(taskStats *TaskStats) StartAt
  • func StartAtFromTestStats(testStats *TestStats) StartAt
• type StatsFilter
  • func (f *StatsFilter) ValidateForTasks() error
  • func (f *StatsFilter) ValidateForTests() error
• type StatsStatus
  • func GetStatsStatus(projectId string) (StatsStatus, error)
• type StatsToUpdate
  • func FindStatsToUpdate(projectID string, requester []string, start, end time.Time) ([]StatsToUpdate, error)
• type TaskStats
  • func GetTaskStats(filter StatsFilter) ([]TaskStats, error)
  • func (s *TaskStats) MarshalBSON() ([]byte, error)
  • func (s *TaskStats) UnmarshalBSON(in []byte) error
• type TestStats
  • func GetTestStats(filter StatsFilter) ([]TestStats, error)
  • func (s *TestStats) MarshalBSON() ([]byte, error)
  • func (s *TestStats) UnmarshalBSON(in []byte) error

Constants

const (
	MaxQueryLimit             = 1001
	GroupByTest       GroupBy = "test"
	GroupByTask       GroupBy = "task"
	GroupByVariant    GroupBy = "variant"
	GroupByDistro     GroupBy = "distro"
	SortEarliestFirst Sort    = "earliest"
	SortLatestFirst   Sort    = "latest"
)

Variables

var (
	// BSON fields for the test stats struct
	TestStatsTestFileKey        = bsonutil.MustHaveTag(TestStats{}, "TestFile")
	TestStatsTaskNameKey        = bsonutil.MustHaveTag(TestStats{}, "TaskName")
	TestStatsBuildVariantKey    = bsonutil.MustHaveTag(TestStats{}, "BuildVariant")
	TestStatsDistroKey          = bsonutil.MustHaveTag(TestStats{}, "Distro")
	TestStatsDateKey            = bsonutil.MustHaveTag(TestStats{}, "Date")
	TestStatsNumPassKey         = bsonutil.MustHaveTag(TestStats{}, "NumPass")
	TestStatsNumFailKey         = bsonutil.MustHaveTag(TestStats{}, "NumFail")
	TestStatsAvgDurationPassKey = bsonutil.MustHaveTag(TestStats{}, "AvgDurationPass")
	TestStatsLastUpdateKey      = bsonutil.MustHaveTag(TestStats{}, "LastUpdate")

	// BSON fields for the task stats struct
	TaskStatsTaskNameKey           = bsonutil.MustHaveTag(TaskStats{}, "TaskName")
	TaskStatsBuildVariantKey       = bsonutil.MustHaveTag(TaskStats{}, "BuildVariant")
	TaskStatsDistroKey             = bsonutil.MustHaveTag(TaskStats{}, "Distro")
	TaskStatsDateKey               = bsonutil.MustHaveTag(TaskStats{}, "Date")
	TaskStatsNumSuccessKey         = bsonutil.MustHaveTag(TaskStats{}, "NumSuccess")
	TaskStatsNumFailedKey          = bsonutil.MustHaveTag(TaskStats{}, "NumFailed")
	TaskStatsNumTotalKey           = bsonutil.MustHaveTag(TaskStats{}, "NumTotal")
	TaskStatsNumTestFailedKey      = bsonutil.MustHaveTag(TaskStats{}, "NumTestFailed")
	TaskStatsNumSetupFailedKey     = bsonutil.MustHaveTag(TaskStats{}, "NumSetupFailed")
	TaskStatsNumSystemFailedKey    = bsonutil.MustHaveTag(TaskStats{}, "NumSystemFailed")
	TaskStatsNumTimeoutKey         = bsonutil.MustHaveTag(TaskStats{}, "NumTimeout")
	TaskStatsAvgDurationSuccessKey = bsonutil.MustHaveTag(TaskStats{}, "AvgDurationSuccess")
	TaskStatsLastUpdateKey         = bsonutil.MustHaveTag(TaskStats{}, "LastUpdate")
)

Functions

func GenerateDailyTaskStats

func GenerateDailyTaskStats(projectId string, requester string, day time.Time, tasks []string, jobRunTime time.Time) error

GenerateDailyTaskStats aggregates the hourly task stats present in the database and saves the resulting daily task stats documents for the project, requester, day, and tasks specified. The day covered is the UTC day corresponding to the given `day` parameter.

func GenerateDailyTestStatsFromHourly

func GenerateDailyTestStatsFromHourly(projectId string, requester string, day time.Time, tasks []string, jobRunTime time.Time) error

GenerateDailyTestStatsFromHourly aggregates the hourly test stats present in the database and saves the resulting daily test stats documents for the project, requester, day, and tasks specified. The day covered is the UTC day corresponding to the given `day` parameter.

func GenerateHourlyTestStats

func GenerateHourlyTestStats(projectId string, requester string, hour time.Time, tasks []string, jobRunTime time.Time) error

GenerateHourlyTestStats aggregates task and testresults prsent in the database and saves the resulting hourly test stats documents for the project, requester, hour, and tasks specified. The hour covered is the UTC hour corresponding to the given `hour` parameter.

func GetDailyTaskDoc

func GetDailyTaskDoc(id DbTaskStatsId) (*dbTaskStats, error)

func GetDailyTestDoc

func GetDailyTestDoc(id DbTestStatsId) (*dbTestStats, error)

func GetHourlyTestDoc

func GetHourlyTestDoc(id DbTestStatsId) (*dbTestStats, error)

func UpdateStatsStatus

func UpdateStatsStatus(projectId string, lastJobRun time.Time, processedTasksUntil time.Time) error

UpdateStatsStatus updates the status of the stats pre-computations for a project.

Types

type DbTaskStatsId

type DbTaskStatsId struct {
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Project      string    `bson:"project"`
	Requester    string    `bson:"requester"`
	Date         time.Time `bson:"date"`
}

DbTaskStatsId represents the _id field for daily_task_stats documents.

type DbTestStatsId

type DbTestStatsId struct {
	TestFile     string    `bson:"test_file"`
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Project      string    `bson:"project"`
	Requester    string    `bson:"requester"`
	Date         time.Time `bson:"date"`
}

DbTestStatsId represents the _id field for hourly_test_stats and daily_test_stats documents.

type GroupBy

type GroupBy string

type Sort

type Sort string

type StartAt

type StartAt struct {
	Date         time.Time
	BuildVariant string
	Task         string
	Test         string
	Distro       string
}

StartAt represents parameters that allow a search query to resume at a specific point. Used for pagination.

func StartAtFromTaskStats

func StartAtFromTaskStats(taskStats *TaskStats) StartAt

StartAtFromTaskStats creates a StartAt that can be used to resume a task stats query. Using the returned StartAt the given TaskStats will be the first result.

func StartAtFromTestStats

func StartAtFromTestStats(testStats *TestStats) StartAt

StartAtFromTestStats creates a StartAt that can be used to resume a test stats query. Using the returned StartAt the given TestStats will be the first result.

type StatsFilter

type StatsFilter struct {
	Project    string
	Requesters []string
	AfterDate  time.Time
	BeforeDate time.Time

	Tests         []string
	Tasks         []string
	BuildVariants []string
	Distros       []string

	GroupNumDays int
	GroupBy      GroupBy
	StartAt      *StartAt
	Limit        int
	Sort         Sort
}

StatsFilter represents search and aggregation parameters when querying the test or task statistics.

func (*StatsFilter) ValidateForTasks

func (f *StatsFilter) ValidateForTasks() error

use with test stats validates that the StatsFilter struct is valid for use with task stats.

func (*StatsFilter) ValidateForTests

func (f *StatsFilter) ValidateForTests() error

validateForTests validates that the StatsFilter struct is valid for use with test stats.

type StatsStatus

type StatsStatus struct {
	ProjectId           string    `bson:"_id"`
	LastJobRun          time.Time `bson:"last_job_run"`
	ProcessedTasksUntil time.Time `bson:"processed_tasks_until"`
}

StatsStatus represents the status for stats pre-computations for a project. LastJobRun is the start date of the last successful pre-computation job that ran for the project. ProcessedTasksUntil is the date before which all finished tasks have been processed. It is usually the same as LastJobRun unless a previous job has failed and the stats computation has not caught up yet.

func GetStatsStatus

func GetStatsStatus(projectId string) (StatsStatus, error)

GetStatsStatus retrieves the status of the stats pre-computations for a project.

type StatsToUpdate

type StatsToUpdate struct {
	ProjectId string    `bson:"project"`
	Requester string    `bson:"requester"`
	Hour      time.Time `bson:"date"`
	Day       time.Time `bson:"day"`
	Tasks     []string  `bson:"task_names"`
}

func FindStatsToUpdate

func FindStatsToUpdate(projectID string, requester []string, start, end time.Time) ([]StatsToUpdate, error)

FidnStatsToUpdate finds the stats that need to be updated as a result of tasks finishing between 'start' and 'end'. The results are ordered by project id, then hour, then requester.

type TaskStats

type TaskStats struct {
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Date         time.Time `bson:"date"`

	NumTotal           int       `bson:"num_total"`
	NumSuccess         int       `bson:"num_success"`
	NumFailed          int       `bson:"num_failed"`
	NumTimeout         int       `bson:"num_timeout"`
	NumTestFailed      int       `bson:"num_test_failed"`
	NumSystemFailed    int       `bson:"num_system_failed"`
	NumSetupFailed     int       `bson:"num_setup_failed"`
	AvgDurationSuccess float64   `bson:"avg_duration_success"`
	LastUpdate         time.Time `bson:"last_update"`
}

TaskStats represents task execution statistics.

func GetTaskStats

func GetTaskStats(filter StatsFilter) ([]TaskStats, error)

GetTaskStats queries the precomputed task statistics using a filter.

func (*TaskStats) MarshalBSON

func (s *TaskStats) MarshalBSON() ([]byte, error)

func (*TaskStats) UnmarshalBSON

func (s *TaskStats) UnmarshalBSON(in []byte) error

type TestStats

type TestStats struct {
	TestFile     string    `bson:"test_file"`
	TaskName     string    `bson:"task_name"`
	BuildVariant string    `bson:"variant"`
	Distro       string    `bson:"distro"`
	Date         time.Time `bson:"date"`

	NumPass         int       `bson:"num_pass"`
	NumFail         int       `bson:"num_fail"`
	AvgDurationPass float64   `bson:"avg_duration_pass"`
	LastUpdate      time.Time `bson:"last_update"`
}

TestStats represents test execution statistics.

func GetTestStats

func GetTestStats(filter StatsFilter) ([]TestStats, error)

GetTestStats queries the precomputed test statistics using a filter.

func (*TestStats) MarshalBSON

func (s *TestStats) MarshalBSON() ([]byte, error)

func (*TestStats) UnmarshalBSON

func (s *TestStats) UnmarshalBSON(in []byte) error