db

package
v0.0.0-...-a96dc21 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Dec 18, 2024 License: BSD-3-Clause Imports: 26 Imported by: 18

Documentation

Index

Constants

View Source
const (
	// Retries attempted by UpdateTasksWithRetries.
	NUM_RETRIES = 5

	// SearchResultLimit is the maximum number of results returned by SearchJobs
	// or SearchTasks.
	SearchResultLimit = 500
)
View Source
const (
	TS_RESOLUTION = time.Microsecond
)

Variables

View Source
var (
	ErrAlreadyExists    = errors.New("Object already exists and modification not allowed.")
	ErrConcurrentUpdate = errors.New("Concurrent update")
	ErrNotFound         = errors.New("Task/Job with given ID does not exist")
	ErrTooManyUsers     = errors.New("Too many users")
	ErrUnknownId        = types.ErrUnknownId
	ErrDoneSearching    = errors.New("Done searching")
)
View Source
var AssertDeepEqual func(t sktest.TestingT, expected, actual interface{})

AssertDeepEqual does a deep equals comparison using the testutils.TestingT interface.

Callers of these tests utils should assign a value to AssertDeepEqual beforehand, e.g.:

AssertDeepEqual = assertdeep.Equal

This is necessary to break the hard linking of this file to the "testing" module.

Functions

func FilterJobs

func FilterJobs(jobs []*types.Job, p *JobSearchParams) []*types.Job

FilterJobs filters the given slice of Jobs to those which match the given search parameters. Provided for use by implementations of JobReader.SearchJobs.

func FilterTasks

func FilterTasks(tasks []*types.Task, p *TaskSearchParams) []*types.Task

FilterTasks filters the given slice of Tasks to those which match the given search parameters. Provided for use by implementations of TaskReader.SearchTasks.

func GetJobsFromWindow

func GetJobsFromWindow(ctx context.Context, db JobReader, w window.Window) ([]*types.Job, error)

GetJobsFromWindow returns all jobs matching the given Window from the JobReader.

func GetTasksFromWindow

func GetTasksFromWindow(ctx context.Context, db TaskReader, w window.Window) ([]*types.Task, error)

GetTasksFromWindow returns all tasks matching the given Window from the TaskReader.

func IsAlreadyExists

func IsAlreadyExists(e error) bool

func IsConcurrentUpdate

func IsConcurrentUpdate(e error) bool

func IsNotFound

func IsNotFound(e error) bool

func IsTooManyUsers

func IsTooManyUsers(e error) bool

func IsUnknownId

func IsUnknownId(e error) bool

func MatchJob

func MatchJob(j *types.Job, p *JobSearchParams) bool

MatchJob returns true if the given Job matches the given search parameters.

func MatchTask

func MatchTask(t *types.Task, p *TaskSearchParams) bool

MatchTask returns true if the given Task matches the given search parameters.

func SearchBoolEqual

func SearchBoolEqual(search *bool, test bool) bool

SearchBoolEqual compares the two bools and returns true if the first is nil or equal to the second.

func SearchInt64Equal

func SearchInt64Equal(search *int64, test int64) bool

SearchInt64Equal compares the two int64s and returns true if the first is either nil or equal to the second.

func SearchStatusEqual

func SearchStatusEqual(search *string, test string) bool

SearchStatusEqual compares the two strings and returns true if the first is either not provided or equal to the second.

func SearchStringEqual

func SearchStringEqual(search *string, test string) bool

SearchStringEqual compares the two strings and returns true if the first is either not provided or equal to the second.

func TestCommentDB

func TestCommentDB(t sktest.TestingT, db CommentDB)

TestCommentDB validates that db correctly implements the CommentDB interface.

func TestJobDB

func TestJobDB(t sktest.TestingT, db JobDB)

TestJobDB performs basic tests on an implementation of JobDB.

func TestJobDBConcurrentUpdate

func TestJobDBConcurrentUpdate(t sktest.TestingT, db JobDB)

Test that PutJob and PutJobs return ErrConcurrentUpdate when a cached Job has been updated in the DB.

func TestJobDBSearch

func TestJobDBSearch(t sktest.TestingT, db JobDB)

TestJobDBSearch tests JobReader.SearchJobs.

func TestModifiedCommitCommentsCh

func TestModifiedCommitCommentsCh(t sktest.TestingT, db DB)

func TestModifiedJobsCh

func TestModifiedJobsCh(t sktest.TestingT, db DB)

func TestModifiedTaskCommentsCh

func TestModifiedTaskCommentsCh(t sktest.TestingT, db DB)

func TestModifiedTaskSpecCommentsCh

func TestModifiedTaskSpecCommentsCh(t sktest.TestingT, db DB)

func TestModifiedTasksCh

func TestModifiedTasksCh(t sktest.TestingT, db DB)

func TestTaskDB

func TestTaskDB(t sktest.TestingT, db TaskDB)

TestTaskDB performs basic tests for an implementation of TaskDB.

func TestTaskDBConcurrentUpdate

func TestTaskDBConcurrentUpdate(t sktest.TestingT, db TaskDB)

Test that PutTask and PutTasks return ErrConcurrentUpdate when a cached Task has been updated in the DB.

func TestTaskDBGetTasksFromDateRangeByRepo

func TestTaskDBGetTasksFromDateRangeByRepo(t sktest.TestingT, db TaskDB)

func TestTaskDBGetTasksFromWindow

func TestTaskDBGetTasksFromWindow(t sktest.TestingT, db TaskDB)

func TestTaskDBSearch

func TestTaskDBSearch(t sktest.TestingT, db TaskDB)

TestTaskDBSearch tests TaskReader.SearchTasks.

func TestUpdateDBFromSwarmingTask

func TestUpdateDBFromSwarmingTask(t sktest.TestingT, db TaskDB)

func TestUpdateDBFromSwarmingTaskTryJob

func TestUpdateDBFromSwarmingTaskTryJob(t sktest.TestingT, db TaskDB)

func TestUpdateTasksWithRetries

func TestUpdateTasksWithRetries(t sktest.TestingT, db TaskDB)

Test UpdateTasksWithRetries and UpdateTaskWithRetries.

func UpdateDBFromTaskResult

func UpdateDBFromTaskResult(ctx context.Context, db TaskDB, res *types.TaskResult) (bool, error)

UpdateDBFromTaskResult updates a task in db from data in s.

func UpdateTaskWithRetries

func UpdateTaskWithRetries(ctx context.Context, db TaskDB, id string, f func(*types.Task) error) (*types.Task, error)

UpdateTaskWithRetries reads, updates, and writes a single Task in the DB. It:

  1. reads the task with the given id,
  2. calls f on that task, and
  3. calls db.PutTask() on the updated task
  4. repeats from step 1 as long as PutTasks returns ErrConcurrentUpdate and retries have not been exhausted.

Returns the updated task if it was successfully updated in the DB. Immediately returns ErrNotFound if db.GetTaskById(id) returns nil. Immediately returns any error returned from f or from PutTasks (except ErrConcurrentUpdate). Returns ErrConcurrentUpdate if retries are exhausted.

func UpdateTasksWithRetries

func UpdateTasksWithRetries(ctx context.Context, db TaskDB, f func() ([]*types.Task, error)) ([]*types.Task, error)

UpdateTasksWithRetries wraps a call to db.PutTasks with retries. It calls db.PutTasks(f()) repeatedly until one of the following happen:

  • f or db.PutTasks returns an error, which is then returned from UpdateTasksWithRetries;
  • PutTasks succeeds, in which case UpdateTasksWithRetries returns the updated Tasks returned by f;
  • retries are exhausted, in which case UpdateTasksWithRetries returns ErrConcurrentUpdate.

Within f, tasks should be refreshed from the DB, e.g. with db.GetModifiedTasks or db.GetTaskById.

Types

type CommentDB

type CommentDB interface {
	// GetComments returns all comments for the given repos.
	//
	// If from is specified, it is a hint that TaskComments and CommitComments
	// before this time will be ignored by the caller, thus they may be ommitted.
	GetCommentsForRepos(ctx context.Context, repos []string, from time.Time) ([]*types.RepoComments, error)

	// PutTaskComment inserts the TaskComment into the database. May return
	// ErrAlreadyExists.
	PutTaskComment(context.Context, *types.TaskComment) error

	// DeleteTaskComment deletes the matching TaskComment from the database.
	// Non-ID fields of the argument are ignored.
	DeleteTaskComment(context.Context, *types.TaskComment) error

	// PutTaskSpecComment inserts the TaskSpecComment into the database. May
	// return ErrAlreadyExists.
	PutTaskSpecComment(context.Context, *types.TaskSpecComment) error

	// DeleteTaskSpecComment deletes the matching TaskSpecComment from the
	// database. Non-ID fields of the argument are ignored.
	DeleteTaskSpecComment(context.Context, *types.TaskSpecComment) error

	// PutCommitComment inserts the CommitComment into the database. May return
	// ErrAlreadyExists.
	PutCommitComment(context.Context, *types.CommitComment) error

	// DeleteCommitComment deletes the matching CommitComment from the database.
	// Non-ID fields of the argument are ignored.
	DeleteCommitComment(context.Context, *types.CommitComment) error

	// ModifiedTaskCommentsCh returns a channel which produces TaskComments
	// as they are modified in the DB. The channel is closed when the given
	// Context is canceled. At least one (possibly empty) item will be sent
	// on the returned channel.
	ModifiedTaskCommentsCh(context.Context) <-chan []*types.TaskComment

	// ModifiedTaskSpecCommentsCh returns a channel which produces
	// TaskSpecComments as they are modified in the DB. The channel is
	// closed when the given Context is canceled. At least one (possibly
	// empty) item will be sent on the returned channel.
	ModifiedTaskSpecCommentsCh(context.Context) <-chan []*types.TaskSpecComment

	// ModifiedCommitCommentsCh returns a channel which produces
	// CommitComments as they are modified in the DB. The channel is closed
	// when the given Context is canceled. At least one (possibly empty)
	// item will be sent on the returned channel.
	ModifiedCommitCommentsCh(context.Context) <-chan []*types.CommitComment
}

CommentDB stores comments on Tasks, TaskSpecs, and commits.

Clients must be tolerant of comments that refer to nonexistent Tasks, TaskSpecs, or commits.

type DB

type DB interface {
	TaskDB
	JobDB
	CommentDB
}

DB implements TaskDB, JobDB, and CommentDB.

func NewDB

func NewDB(tdb TaskDB, jdb JobDB, cdb CommentDB) DB

NewDB returns a DB that delegates to independent TaskDB, JobDB, and CommentDB.

type DBCloser

type DBCloser interface {
	io.Closer
	DB
}

DBCloser is a DB that must be closed when no longer in use.

type JobDB

type JobDB interface {
	JobReader

	// PutJob inserts or updates the Job in the database. Job's Id field
	// must be empty if it is a new Job. PutJob will set Job.DbModified.
	PutJob(context.Context, *types.Job) error

	// PutJobs inserts or updates the Jobs in the database. Each Jobs' Id
	// field must be empty if it is a new Job. Each Jobs' DbModified field
	// will be set. All modifications are performed in a single transaction;
	// the caller must determine what to do when there are too many Jobs to
	// be inserted at once for a given JobDB implementation. Use only when
	// consistency is important; otherwise, callers should use
	// PutJobsInChunks.
	PutJobs(context.Context, []*types.Job) error

	// PutJobsInChunks is like PutJobs but inserts Jobs in multiple
	// transactions. Not appropriate for updates in which consistency is
	// important.
	PutJobsInChunks(context.Context, []*types.Job) error
}

JobDB is used by the task scheduler to store Jobs.

type JobReader

type JobReader interface {
	// GetJobById returns the job with the given Id field. Returns nil, nil if
	// job is not found.
	GetJobById(context.Context, string) (*types.Job, error)

	// GetJobsFromDateRange retrieves all jobs with Created in the given
	// range. The returned jobs are sorted by Created timestamp. The string
	// field is an optional repository; if provided, only return tasks
	// associated with that repo.
	GetJobsFromDateRange(context.Context, time.Time, time.Time, string) ([]*types.Job, error)

	// ModifiedJobsCh returns a channel which produces Jobs as they are
	// modified in the DB. The channel is closed when the given Context is
	// canceled. The channel will immediately produce a slice of Jobs which
	// may or may not be empty.
	ModifiedJobsCh(context.Context) <-chan []*types.Job

	// SearchJobs retrieves all matching Jobs from the DB.
	SearchJobs(context.Context, *JobSearchParams) ([]*types.Job, error)
}

JobReader is a read-only view of a JobDB.

type JobSearchParams

type JobSearchParams struct {
	BuildbucketBuildID *int64           `json:"buildbucket_build_id,string,omitempty"`
	IsForce            *bool            `json:"is_force,omitempty"`
	Issue              *string          `json:"issue,omitempty"`
	Name               *string          `json:"name"`
	Patchset           *string          `json:"patchset,omitempty"`
	Repo               *string          `json:"repo,omitempty"`
	Revision           *string          `json:"revision,omitempty"`
	Status             *types.JobStatus `json:"status"`
	TimeStart          *time.Time       `json:"time_start"`
	TimeEnd            *time.Time       `json:"time_end"`
}

JobSearchParams are parameters on which Jobs may be searched. All fields are optional; if a field is not provided, the search will return Jobs with any value for that field. If TimeEnd is not provided the search terminates at the current time. If TimeStart is not provided the search begins 24 hours before TimeEnd.

type RemoteDB

type RemoteDB interface {
	TaskReader
	JobReader
	CommentDB
}

RemoteDB allows retrieving tasks and jobs and full access to comments.

type TaskDB

type TaskDB interface {
	TaskReader

	// AssignId sets the given task's Id field. Does not insert the task into the
	// database.
	AssignId(context.Context, *types.Task) error

	// PutTask inserts or updates the Task in the database. Task's Id field must
	// be empty or set with AssignId. PutTask will set Task.DbModified.
	PutTask(context.Context, *types.Task) error

	// PutTasks inserts or updates the Tasks in the database. Each Task's Id field
	// must be empty or set with AssignId. Each Task's DbModified field will be
	// set. All modifications are performed in a single transaction; the
	// caller must determine what to do when there are too many Tasks to be
	// inserted at once for a given TaskDB implementation. Use only when
	// consistency is important; otherwise, callers should use
	// PutTasksInChunks.
	PutTasks(context.Context, []*types.Task) error

	// PutTasksInChunks is like PutTasks but inserts Tasks in multiple
	// transactions. Not appropriate for updates in which consistency is
	// important.
	PutTasksInChunks(context.Context, []*types.Task) error
}

TaskDB is used by the task scheduler to store Tasks.

type TaskReader

type TaskReader interface {
	// GetTaskById returns the task with the given Id field. Returns nil, nil if
	// task is not found.
	GetTaskById(context.Context, string) (*types.Task, error)

	// GetTasksFromDateRange retrieves all tasks with Created in the given range.
	// The returned tasks are sorted by Created timestamp. The string field is
	// an optional repository; if provided, only return tasks associated with
	// that repo.
	GetTasksFromDateRange(context.Context, time.Time, time.Time, string) ([]*types.Task, error)

	// ModifiedTasksCh returns a channel which produces Tasks as they are
	// modified in the DB. The channel is closed when the given Context is
	// canceled. The channel will immediately produce a slice of Tasks which
	// may or may not be empty.
	ModifiedTasksCh(context.Context) <-chan []*types.Task

	// SearchTasks retrieves all matching Tasks from the DB.
	SearchTasks(context.Context, *TaskSearchParams) ([]*types.Task, error)
}

TaskReader is a read-only view of a TaskDB.

type TaskSearchParams

type TaskSearchParams struct {
	Attempt     *int64            `json:"attempt,string,omitempty"`
	Status      *types.TaskStatus `json:"status"`
	ForcedJobId *string           `json:"forcedJobId,omitempty"`
	Issue       *string           `json:"issue,omitempty"`
	Name        *string           `json:"name"`
	Patchset    *string           `json:"patchset,omitempty"`
	Repo        *string           `json:"repo,omitempty"`
	Revision    *string           `json:"revision,omitempty"`
	TimeStart   *time.Time        `json:"time_start"`
	TimeEnd     *time.Time        `json:"time_end"`
}

TaskSearchParams are parameters on which Tasks may be searched. All fields are optional; if a field is not provided, the search will return Tasks with any value for that field. If TimeEnd is not provided the search terminates at the current time. If TimeStart is not provided the search begins 24 hours before TimeEnd.

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL