storage

package
v0.0.0-...-a69e935 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 1, 2019 License: Apache-2.0 Imports: 11 Imported by: 77

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type ErrorMetrics 

type ErrorMetrics struct {
	ReadFailure        tally.Counter
	WriteFailure       tally.Counter
	AlreadyExists      tally.Counter
	ReadTimeout        tally.Counter
	WriteTimeout       tally.Counter
	RequestUnavailable tally.Counter
	TooManyTimeouts    tally.Counter
	ConnUnavailable    tally.Counter
	SessionClosed      tally.Counter
	NoConnections      tally.Counter
	ConnectionClosed   tally.Counter
	NoStreams          tally.Counter
	NotTransient       tally.Counter
	CASNotApplied      tally.Counter
}

ErrorMetrics is a struct for tracking all the storage error counters

type FrameworkInfoStore 

type FrameworkInfoStore interface {
	SetMesosStreamID(ctx context.Context, frameworkName string, mesosStreamID string) error
	SetMesosFrameworkID(ctx context.Context, frameworkName string, frameworkID string) error
	GetMesosStreamID(ctx context.Context, frameworkName string) (string, error)
	GetFrameworkID(ctx context.Context, frameworkName string) (string, error)
}

FrameworkInfoStore is the interface to store mesosStreamID for peloton frameworks

type FrameworkStoreMetrics 

type FrameworkStoreMetrics struct {
	FrameworkUpdate     tally.Counter
	FrameworkUpdateFail tally.Counter
	FrameworkIDGet      tally.Counter
	FrameworkIDGetFail  tally.Counter
	StreamIDGet         tally.Counter
	StreamIDGetFail     tally.Counter
}

FrameworkStoreMetrics is a struct for tracking framework and streamID related counters in the storage layer

type JobMetrics 

type JobMetrics struct {
	JobCreate           tally.Counter
	JobCreateConfig     tally.Counter
	JobCreateFail       tally.Counter
	JobCreateConfigFail tally.Counter

	JobCreateRuntime     tally.Counter
	JobCreateRuntimeFail tally.Counter

	JobUpdate     tally.Counter
	JobUpdateFail tally.Counter

	JobGet      tally.Counter
	JobGetFail  tally.Counter
	JobNotFound tally.Counter

	JobQuery     tally.Counter
	JobQueryAll  tally.Counter
	JobQueryFail tally.Counter

	JobDelete     tally.Counter
	JobDeleteFail tally.Counter

	JobGetRuntime     tally.Counter
	JobGetRuntimeFail tally.Counter

	JobGetAll     tally.Counter
	JobGetAllFail tally.Counter

	JobGetByRespoolID     tally.Counter
	JobGetByRespoolIDFail tally.Counter

	JobUpdateRuntime     tally.Counter
	JobUpdateRuntimeFail tally.Counter

	JobUpdateConfig     tally.Counter
	JobUpdateConfigFail tally.Counter

	JobUpdateInfo     tally.Counter
	JobUpdateInfoFail tally.Counter

	JobNameToID     tally.Counter
	JobNameToIDFail tally.Counter

	JobGetNameToID     tally.Counter
	JobGetNameToIDFail tally.Counter
}

JobMetrics is a struct for tracking all the job related counters in the storage layer

type JobStore 

type JobStore interface {
	QueryJobs(ctx context.Context, respoolID *peloton.ResourcePoolID, spec *job.QuerySpec, summaryOnly bool) ([]*job.JobInfo, []*job.JobSummary, uint32, error)
	// DeleteJob deletes the job configuration, runtime
	// and all tasks in DB of a given job
	DeleteJob(ctx context.Context, jobID string) error
	// GetMaxJobConfigVersion returns the maximum version of configs of a given job
	GetMaxJobConfigVersion(ctx context.Context, jobID string) (uint64, error)
}

JobStore is the interface to store job states TODO: Move all arguments from proto pointers to golang data types

type Metrics 

type Metrics struct {
	JobMetrics                *JobMetrics
	TaskMetrics               *TaskMetrics
	UpdateMetrics             *UpdateMetrics
	ResourcePoolMetrics       *ResourcePoolMetrics
	FrameworkStoreMetrics     *FrameworkStoreMetrics
	VolumeMetrics             *VolumeMetrics
	ErrorMetrics              *ErrorMetrics
	WorkflowMetrics           *WorkflowMetrics
	OrmJobMetrics             *OrmJobMetrics
	OrmRespoolMetrics         *OrmRespoolMetrics
	OrmTaskMetrics            *OrmTaskMetrics
	OrmHostInfoMetrics        *OrmHostInfoMetrics
	OrmJobUpdateEventsMetrics *OrmJobUpdateEventsMetrics
}

Metrics is a struct for tracking all the general purpose counters that have relevance to the storage layer, i.e. how many jobs and tasks were created/deleted in the storage layer

func NewMetrics 

func NewMetrics(scope tally.Scope) *Metrics

NewMetrics returns a new Metrics struct, with all metrics initialized and rooted at the given tally.Scope

type OrmHostInfoMetrics 

type OrmHostInfoMetrics struct {
	HostInfoAdd     tally.Counter
	HostInfoAddFail tally.Counter

	HostInfoGet     tally.Counter
	HostInfoGetFail tally.Counter

	HostInfoGetAll     tally.Counter
	HostInfoGetAllFail tally.Counter

	HostInfoUpdate     tally.Counter
	HostInfoUpdateFail tally.Counter

	HostInfoCurrentPoolUpdate     tally.Counter
	HostInfoCurrentPoolUpdateFail tally.Counter

	HostInfoDesiredPoolUpdate     tally.Counter
	HostInfoDesiredPoolUpdateFail tally.Counter

	HostInfoDelete     tally.Counter
	HostInfoDeleteFail tally.Counter

	HostInfoCompareAndSet     tally.Counter
	HostInfoCompareAndSetFail tally.Counter
}

OrmHostInfoMetrics tracks counters for host info related table

type OrmJobMetrics 

type OrmJobMetrics struct {
	// job_index
	JobIndexCreate     tally.Counter
	JobIndexCreateFail tally.Counter
	JobIndexGet        tally.Counter
	JobIndexGetFail    tally.Counter
	JobIndexGetAll     tally.Counter
	JobIndexGetAllFail tally.Counter
	JobIndexUpdate     tally.Counter
	JobIndexUpdateFail tally.Counter
	JobIndexDelete     tally.Counter
	JobIndexDeleteFail tally.Counter

	// job_name_to_id
	JobNameToIDCreate     tally.Counter
	JobNameToIDCreateFail tally.Counter
	JobNameToIDGetAll     tally.Counter
	JobNameToIDGetAllFail tally.Counter

	// job_config
	JobConfigCreate     tally.Counter
	JobConfigCreateFail tally.Counter
	JobConfigGet        tally.Counter
	JobConfigGetFail    tally.Counter
	JobConfigDelete     tally.Counter
	JobConfigDeleteFail tally.Counter

	// active_jobs.
	ActiveJobsCreate         tally.Counter
	ActiveJobsCreateFail     tally.Counter
	ActiveJobsGetAll         tally.Counter
	ActiveJobsGetAllFail     tally.Counter
	ActiveJobsGetAllDuration tally.Timer
	ActiveJobsDelete         tally.Counter
	ActiveJobsDeleteFail     tally.Counter

	// secret_info
	SecretInfoCreate     tally.Counter
	SecretInfoCreateFail tally.Counter
	SecretInfoGet        tally.Counter
	SecretInfoGetFail    tally.Counter
	SecretInfoUpdate     tally.Counter
	SecretInfoUpdateFail tally.Counter
	SecretInfoDelete     tally.Counter
	SecretInfoDeleteFail tally.Counter
}

OrmJobMetrics tracks counters for job related tables accessed through ORM layer

type OrmJobUpdateEventsMetrics 

type OrmJobUpdateEventsMetrics struct {
	JobUpdateEventsCreate     tally.Counter
	JobUpdateEventsCreateFail tally.Counter
	JobUpdateEventsGet        tally.Counter
	JobUpdateEventsGetFail    tally.Counter
	JobUpdateEventsDelete     tally.Counter
	JobUpdateEventsDeleteFail tally.Counter
}

OrmJobUpdateEventsMetrics tracks counter of job update events related tables

type OrmRespoolMetrics 

type OrmRespoolMetrics struct {
	RespoolCreate     tally.Counter
	RespoolCreateFail tally.Counter
	RespoolGet        tally.Counter
	RespoolGetFail    tally.Counter
	RespoolGetAll     tally.Counter
	RespoolGetAllFail tally.Counter
	RespoolUpdate     tally.Counter
	RespoolUpdateFail tally.Counter
	RespoolDelete     tally.Counter
	RespoolDeleteFail tally.Counter
}

OrmRespoolMetrics tracks counters for resource pools related tables accessed through ORM layer.

type OrmTaskMetrics 

type OrmTaskMetrics struct {
	PodEventsAdd     tally.Counter
	PodEventsAddFail tally.Counter
	PodEventsGet     tally.Counter
	PodEventsGetFail tally.Counter

	TaskConfigV2Create     tally.Counter
	TaskConfigV2CreateFail tally.Counter

	TaskConfigV2Get     tally.Counter
	TaskConfigV2GetFail tally.Counter

	TaskConfigLegacyGet     tally.Counter
	TaskConfigLegacyGetFail tally.Counter

	PodSpecGet     tally.Counter
	PodSpecGetFail tally.Counter
}

OrmTaskMetrics tracks counters for pod related tables

type PersistentVolumeStore 

type PersistentVolumeStore interface {
	CreatePersistentVolume(ctx context.Context, volumeInfo *volume.PersistentVolumeInfo) error
	UpdatePersistentVolume(ctx context.Context, volumeInfo *volume.PersistentVolumeInfo) error
	GetPersistentVolume(ctx context.Context, volumeID *peloton.VolumeID) (*volume.PersistentVolumeInfo, error)
}

PersistentVolumeStore is the interface to store all the persistent volume info

type ResourcePoolMetrics 

type ResourcePoolMetrics struct {
	ResourcePoolCreate     tally.Counter
	ResourcePoolCreateFail tally.Counter

	ResourcePoolUpdate     tally.Counter
	ResourcePoolUpdateFail tally.Counter

	ResourcePoolDelete     tally.Counter
	ResourcePoolDeleteFail tally.Counter

	ResourcePoolGet     tally.Counter
	ResourcePoolGetFail tally.Counter
}

ResourcePoolMetrics is a struct for tracking resource pool related counters in the storage layer

type Store 

type Store interface {
	JobStore
	TaskStore
	UpdateStore
	FrameworkInfoStore
	PersistentVolumeStore
}

Store is is a generic store interface which is a collection of different store interfaces

type TaskMetrics 

type TaskMetrics struct {
	TaskCreate     tally.Counter
	TaskCreateFail tally.Counter

	TaskGet      tally.Counter
	TaskGetFail  tally.Counter
	TaskNotFound tally.Counter

	TaskGetConfig     tally.Counter
	TaskGetConfigFail tally.Counter
	// This metric is to indicate how many task gets are performed using
	// legacy task_config table
	TaskGetConfigLegacy tally.Counter

	TaskGetConfigs     tally.Counter
	TaskGetConfigsFail tally.Counter

	TaskConfigBackFill     tally.Counter
	TaskConfigBackFillFail tally.Counter

	TaskLogState     tally.Counter
	TaskLogStateFail tally.Counter

	TaskGetLogState     tally.Counter
	TaskGetLogStateFail tally.Counter

	TaskGetForJob     tally.Counter
	TaskGetForJobFail tally.Counter

	TaskGetForJobAndStates     tally.Counter
	TaskGetForJobAndStatesFail tally.Counter

	TaskIDsGetForJobAndState     tally.Counter
	TaskIDsGetForJobAndStateFail tally.Counter

	TaskSummaryForJob     tally.Counter
	TaskSummaryForJobFail tally.Counter

	TaskGetForJobRange     tally.Counter
	TaskGetForJobRangeFail tally.Counter

	TaskGetRuntimesForJobRange     tally.Counter
	TaskGetRuntimesForJobRangeFail tally.Counter

	TaskGetRuntime     tally.Counter
	TaskGetRuntimeFail tally.Counter

	TaskUpdateRuntime     tally.Counter
	TaskUpdateRuntimeFail tally.Counter

	TaskDelete     tally.Counter
	TaskDeleteFail tally.Counter

	TaskUpdate     tally.Counter
	TaskUpdateFail tally.Counter

	TaskQueryTasks     tally.Counter
	TaskQueryTasksFail tally.Counter

	PodEventsAddSuccess tally.Counter
	PodEventsAddFail    tally.Counter

	PodEventsGetSucess tally.Counter
	PodEventsGetFail   tally.Counter

	PodEventsDeleteSucess tally.Counter
	PodEventsDeleteFail   tally.Counter
}

TaskMetrics is a struct for tracking all the task related counters in the storage layer

type TaskStore 

type TaskStore interface {
	// CreateTaskRuntime creates the runtime of a given task
	CreateTaskRuntime(
		ctx context.Context,
		id *peloton.JobID,
		instanceID uint32,
		runtime *task.RuntimeInfo,
		createdBy string,
		jobType job.JobType) error
	// GetTaskRuntime gets the runtime of a given task
	GetTaskRuntime(ctx context.Context, jobID *peloton.JobID, instanceID uint32) (*task.RuntimeInfo, error)
	// UpdateTaskRuntime updates the runtime of a given task
	UpdateTaskRuntime(
		ctx context.Context,
		jobID *peloton.JobID,
		instanceID uint32,
		runtime *task.RuntimeInfo,
		jobType job.JobType) error

	// GetTasksForJob gets the task info for all tasks in a job
	GetTasksForJob(ctx context.Context, id *peloton.JobID) (map[uint32]*task.TaskInfo, error)
	// GetTasksForJobAndStates gets the task info for all
	// tasks in a given job and in a given state
	GetTasksForJobAndStates(ctx context.Context, id *peloton.JobID, states []task.TaskState) (map[uint32]*task.TaskInfo, error)
	// GetTaskRuntimesForJobByRange gets the task runtime for all
	// tasks in a job with instanceID in the given range
	GetTaskRuntimesForJobByRange(ctx context.Context, id *peloton.JobID, instanceRange *task.InstanceRange) (map[uint32]*task.RuntimeInfo, error)
	// GetTasksForJobByRange gets the task info for all
	// tasks in a job with instanceID in the given range
	GetTasksForJobByRange(ctx context.Context, id *peloton.JobID, Range *task.InstanceRange) (map[uint32]*task.TaskInfo, error)
	// GetTaskForJob gets the task info for a given task
	GetTaskForJob(ctx context.Context, jobID string, instanceID uint32) (map[uint32]*task.TaskInfo, error)
	// GetTaskConfigs gets the task config for all tasks in a job
	// for all the instanceIDs provided in the input
	GetTaskConfigs(ctx context.Context, id *peloton.JobID, instanceIDs []uint32, version uint64) (map[uint32]*task.TaskConfig, *models.ConfigAddOn, error)
	// GetTaskByID gets the task info for a given task
	GetTaskByID(ctx context.Context, taskID string) (*task.TaskInfo, error)
	// QueryTasks queries for all tasks in a job matching the QuerySpec
	QueryTasks(ctx context.Context, id *peloton.JobID, spec *task.QuerySpec) ([]*task.TaskInfo, uint32, error)
	// DeleteTaskRuntime deletes the task runtime for a given job instance
	DeleteTaskRuntime(ctx context.Context, id *peloton.JobID, instanceID uint32) error
	// DeletePodEvents deletes the pod events for provided JobID, InstanceID and RunID in the range [fromRunID-toRunID)
	DeletePodEvents(ctx context.Context, jobID string, instanceID uint32, fromRunID uint64, toRunID uint64) error
	// GetPodEvents returns pod events for a Job + Instance + PodID (optional), events are sorted descending timestamp order
	GetPodEvents(ctx context.Context, jobID string, instanceID uint32, podID ...string) ([]*pod.PodEvent, error)
}

TaskStore is the interface to store task states TODO: Move all arguments from proto pointers to golang data types

type UpdateMetrics 

type UpdateMetrics struct {
	UpdateCreate     tally.Counter
	UpdateCreateFail tally.Counter

	UpdateGet     tally.Counter
	UpdateGetFail tally.Counter

	UpdateWriteProgress     tally.Counter
	UpdateWriteProgressFail tally.Counter

	UpdateGetProgess     tally.Counter
	UpdateGetProgessFail tally.Counter

	UpdateGetForJob     tally.Counter
	UpdateGetForJobFail tally.Counter

	UpdateDeleteFail tally.Counter
	UpdateDelete     tally.Counter

	JobUpdateEventAdd     tally.Counter
	JobUpdateEventAddFail tally.Counter

	JobUpdateEventGet     tally.Counter
	JobUpdateEventGetFail tally.Counter

	JobUpdateEventDelete     tally.Counter
	JobUpdateEventDeleteFail tally.Counter
}

UpdateMetrics is a struct for tracking job update related counters in the storage layer.

type UpdateStore 

type UpdateStore interface {
	// CreateUpdate by creating a new update in the storage. It's an error
	// if the update already exists.
	CreateUpdate(
		ctx context.Context,
		updateInfo *models.UpdateModel,
	) error

	// DeleteUpdate deletes the update from the update_info table and deletes all
	// job and task configurations created for the update.
	DeleteUpdate(
		ctx context.Context,
		updateID *peloton.UpdateID,
		jobID *peloton.JobID,
		jobConfigVersion uint64,
	) error

	// GetUpdate fetches the job update stored in the DB
	GetUpdate(ctx context.Context, id *peloton.UpdateID) (
		*models.UpdateModel,
		error,
	)

	// WriteUpdateProgress writes the progress of the job update to the DB
	WriteUpdateProgress(
		ctx context.Context,
		updateInfo *models.UpdateModel,
	) error

	// ModifyUpdate modify the progress of an update,
	// instances to update/remove/add and the job config version
	ModifyUpdate(
		ctx context.Context,
		updateInfo *models.UpdateModel,
	) error

	// GetUpdateProgess fetches the job update progress, which includes the
	// instances already updated, instances being updated and the current
	// state of the update.
	GetUpdateProgress(ctx context.Context, id *peloton.UpdateID) (
		*models.UpdateModel,
		error,
	)

	// GetUpdatesForJob returns the list of job updates created for a given job
	GetUpdatesForJob(ctx context.Context, jobID string) ([]*peloton.UpdateID, error)

	// AddWorkflowEvent adds a workflow event for an update and instance
	// to track the progress
	AddWorkflowEvent(
		ctx context.Context,
		updateID *peloton.UpdateID,
		instanceID uint32,
		updateType models.WorkflowType,
		updateState update.State,
	) error

	// GetWorkflowEvents gets workflow events for an update and instance,
	// events are sorted in descending create timestamp
	GetWorkflowEvents(
		ctx context.Context,
		updateID *peloton.UpdateID,
		instanceID uint32,
		limit uint32,
	) ([]*stateless.WorkflowEvent, error)
}

UpdateStore is the interface to store updates and updates progress.

type VolumeMetrics 

type VolumeMetrics struct {
	VolumeCreate     tally.Counter
	VolumeCreateFail tally.Counter
	VolumeUpdate     tally.Counter
	VolumeUpdateFail tally.Counter
	VolumeGet        tally.Counter
	VolumeGetFail    tally.Counter
	VolumeDelete     tally.Counter
	VolumeDeleteFail tally.Counter
}

VolumeMetrics is a struct for tracking disk related counters in the storage layer

type VolumeNotFoundError 

type VolumeNotFoundError struct {
	VolumeID *peloton.VolumeID
}

VolumeNotFoundError indicates that persistent volume is not found

func (*VolumeNotFoundError) Error 

func (e *VolumeNotFoundError) Error() string

type WorkflowMetrics 

type WorkflowMetrics struct {
	WorkflowEventsAdd     tally.Counter
	WorkflowEventsAddFail tally.Counter

	WorkflowEventsGet     tally.Counter
	WorkflowEventsGetFail tally.Counter

	WorkflowEventsDelete     tally.Counter
	WorkflowEventsDeleteFail tally.Counter
}

WorkflowMetrics is a struct for tracking all the workflow operations/events

Source Files

View all Source files

Directories

Path Synopsis
api
Package api provides data access management to structured storage backend a very simple usage example is: err := storage.Initialize(Option{}) defer storage.Close() store, err := storage.OpenDataStore("test_schema_name") queryBuilder := store.NewQuery() statement := queryBuilder.Insert("user").Columns("name", "count").Values("json", 1234) result := store.Execute(statement) allResults, err := result.All()
 Package api provides data access management to structured storage backend a very simple usage example is: err := storage.Initialize(Option{}) defer storage.Close() store, err := storage.OpenDataStore("test_schema_name") queryBuilder := store.NewQuery() statement := queryBuilder.Insert("user").Columns("name", "count").Values("json", 1234) result := store.Execute(statement) allResults, err := result.All()
impl
connectors
cassandra
base
Package querybuilder provides a fluent SQL generator.
 Package querybuilder provides a fluent SQL generator.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.