model

Documentation

Index

• Constants
• type JobSpec
  • func ToJobSpec(protoSpec *pb.JobSpecification) *JobSpec
  • func (j *JobSpec) MergeFrom(anotherJobSpec *JobSpec)
  • func (j *JobSpec) ToProto() *pb.JobSpecification
• type JobSpecBehavior
• type JobSpecBehaviorNotifier
• type JobSpecBehaviorRetry
• type JobSpecBehaviorWebhook
• type JobSpecDependency
• type JobSpecDependencyHTTP
• type JobSpecHook
• type JobSpecMetadata
• type JobSpecMetadataAirflow
• type JobSpecMetadataResource
• type JobSpecMetadataResourceConfig
• type JobSpecSchedule
• type JobSpecTask
• type JobSpecTaskWindow
• type Preset
• type PresetsMap
• type ResourceSpec
  • func (r ResourceSpec) ToProto() (*pb.ResourceSpecification, error)
• type WebhookEndpoint

Constants

const NewWindowVersion = 3

Types

type JobSpec

type JobSpec struct {
	Version      int                 `yaml:"version,omitempty"`
	Name         string              `yaml:"name"`
	Owner        string              `yaml:"owner"`
	Description  string              `yaml:"description,omitempty"`
	Schedule     JobSpecSchedule     `yaml:"schedule"`
	Behavior     JobSpecBehavior     `yaml:"behavior"`
	Task         JobSpecTask         `yaml:"task"`
	Asset        map[string]string   `yaml:"-"`
	Labels       map[string]string   `yaml:"labels,omitempty"`
	Hooks        []JobSpecHook       `yaml:"hooks"`
	Dependencies []JobSpecDependency `yaml:"dependencies"`
	Metadata     *JobSpecMetadata    `yaml:"metadata,omitempty"`
	Path         string              `yaml:"-"`
}

func ToJobSpec

func ToJobSpec(protoSpec *pb.JobSpecification) *JobSpec

func (*JobSpec) MergeFrom

func (j *JobSpec) MergeFrom(anotherJobSpec *JobSpec)

TODO: there are some refactors required, however it will be addressed once we relook at the job spec inheritance

func (*JobSpec) ToProto

func (j *JobSpec) ToProto() *pb.JobSpecification

type JobSpecBehavior

type JobSpecBehavior struct {
	DependsOnPast bool                      `yaml:"depends_on_past"`
	Catchup       bool                      `yaml:"catch_up,omitempty"`
	Retry         *JobSpecBehaviorRetry     `yaml:"retry,omitempty"`
	Notify        []JobSpecBehaviorNotifier `yaml:"notify,omitempty"`
	Webhook       []JobSpecBehaviorWebhook  `yaml:"webhook,omitempty"`
}

type JobSpecBehaviorNotifier

type JobSpecBehaviorNotifier struct {
	On       string            `yaml:"on"`
	Config   map[string]string `yaml:"config"`
	Channels []string          `yaml:"channels"`
}

type JobSpecBehaviorRetry

type JobSpecBehaviorRetry struct {
	Count              int           `yaml:"count,omitempty"`
	Delay              time.Duration `yaml:"delay,omitempty"`
	ExponentialBackoff bool          `yaml:"exponential_backoff,omitempty"`
}

type JobSpecBehaviorWebhook

type JobSpecBehaviorWebhook struct {
	On        string            `yaml:"on"`
	Endpoints []WebhookEndpoint `yaml:"endpoints"`
}

type JobSpecDependency

type JobSpecDependency struct {
	JobName string                 `yaml:"job,omitempty"`
	Type    string                 `yaml:"type,omitempty"`
	HTTP    *JobSpecDependencyHTTP `yaml:"http,omitempty"`
}

type JobSpecDependencyHTTP

type JobSpecDependencyHTTP struct {
	Name          string            `yaml:"name"`
	RequestParams map[string]string `yaml:"params,omitempty"`
	URL           string            `yaml:"url"`
	Headers       map[string]string `yaml:"headers,omitempty"`
}

type JobSpecHook

type JobSpecHook struct {
	Name   string            `yaml:"name"`
	Config map[string]string `yaml:"config,omitempty"`
}

type JobSpecMetadata

type JobSpecMetadata struct {
	Resource *JobSpecMetadataResource `yaml:"resource,omitempty"`
	Airflow  *JobSpecMetadataAirflow  `yaml:"airflow,omitempty"`
}

type JobSpecMetadataAirflow

type JobSpecMetadataAirflow struct {
	Pool  string `yaml:"pool" json:"pool"`
	Queue string `yaml:"queue" json:"queue"`
}

type JobSpecMetadataResource

type JobSpecMetadataResource struct {
	Request *JobSpecMetadataResourceConfig `yaml:"request,omitempty"`
	Limit   *JobSpecMetadataResourceConfig `yaml:"limit,omitempty"`
}

type JobSpecMetadataResourceConfig

type JobSpecMetadataResourceConfig struct {
	Memory string `yaml:"memory,omitempty"`
	CPU    string `yaml:"cpu,omitempty"`
}

type JobSpecSchedule

type JobSpecSchedule struct {
	StartDate string `yaml:"start_date"`
	EndDate   string `yaml:"end_date,omitempty"`
	Interval  string `yaml:"interval"`
}

type JobSpecTask

type JobSpecTask struct {
	Name   string            `yaml:"name"`
	Config map[string]string `yaml:"config,omitempty"`
	Window JobSpecTaskWindow `yaml:"window,omitempty"`
}

type JobSpecTaskWindow

type JobSpecTaskWindow struct {
	Size string `yaml:"size,omitempty"`
	// deprecated, replaced by ShiftBy
	Offset     string `yaml:"offset,omitempty"`
	TruncateTo string `yaml:"truncate_to,omitempty"`
	Preset     string `yaml:"preset,omitempty"`
	ShiftBy    string `yaml:"shift_by,omitempty"`
	Location   string `yaml:"location,omitempty"`
}

type Preset

type Preset struct {
	Description string            `yaml:"description"`
	Window      JobSpecTaskWindow `yaml:"window"`
}

type PresetsMap

type PresetsMap struct {
	Presets map[string]Preset `yaml:"presets"`
}

type ResourceSpec

type ResourceSpec struct {
	Version int                    `yaml:"version"`
	Name    string                 `yaml:"name"`
	Type    string                 `yaml:"type"`
	Labels  map[string]string      `yaml:"labels"`
	Spec    map[string]interface{} `yaml:"spec"`
	Path    string                 `yaml:"-"`
}

func (ResourceSpec) ToProto

func (r ResourceSpec) ToProto() (*pb.ResourceSpecification, error)

type WebhookEndpoint

type WebhookEndpoint struct {
	URL     string            `yaml:"url"`
	Headers map[string]string `yaml:"headers,omitempty"`
}