scheduler

Documentation

Overview

Schedule tasks to run on available resources assigned by master.

Index

• func Assign(master string, request *pb.ComputeRequest) (*pb.AllocationResult, error)
• func NewDeleteDatasetShardRequest(name string) *msg.ControlMessage
• func NewGetStatusRequest(requestId uint32) *msg.ControlMessage
• func NewStartRequest(name string, dir string, instructions *msg.InstructionSet, ...) *msg.ControlMessage
• func NewStopRequest(requestId uint32) *msg.ControlMessage
• func RemoteDirectCommand(server string, command *msg.ControlMessage) (response *msg.ControlMessage, err error)
• func RemoteDirectExecute(server string, command *msg.ControlMessage) error
• type DatasetShardLocator
  • func NewDatasetShardLocator() *DatasetShardLocator
  • func (l *DatasetShardLocator) GetShardLocation(shardName string) (pb.DataLocation, bool)
  • func (l *DatasetShardLocator) SetShardLocation(name string, location pb.DataLocation)
• type ReleaseTaskGroupInputs
• type RemoteExecutorStatus
• type Scheduler
  • func NewScheduler(leader string, option *SchedulerOption) *Scheduler
  • func (s *Scheduler) EventLoop()
  • func (s *Scheduler) Fetch(demands []market.Demand)
  • func (s *Scheduler) GetShardLocation(shard *flow.DatasetShard) (pb.DataLocation, bool)
  • func (s *Scheduler) Score(r market.Requirement, bid float64, obj market.Object) float64
  • func (s *Scheduler) SetShardLocation(shard *flow.DatasetShard, loc pb.DataLocation)
• type SchedulerOption
• type SubmitTaskGroup
• type TaskGroupStatus

Functions

func Assign

func Assign(master string, request *pb.ComputeRequest) (*pb.AllocationResult, error)

func NewDeleteDatasetShardRequest

func NewDeleteDatasetShardRequest(name string) *msg.ControlMessage

func NewGetStatusRequest

func NewGetStatusRequest(requestId uint32) *msg.ControlMessage

func NewStartRequest

func NewStartRequest(name string, dir string, instructions *msg.InstructionSet,
	allocated *pb.ComputeResource, envs []string, host string, port int32) *msg.ControlMessage

func NewStopRequest

func NewStopRequest(requestId uint32) *msg.ControlMessage

func RemoteDirectCommand

func RemoteDirectCommand(server string, command *msg.ControlMessage) (response *msg.ControlMessage, err error)

func RemoteDirectExecute

func RemoteDirectExecute(server string, command *msg.ControlMessage) error

Types

type DatasetShardLocator

type DatasetShardLocator struct {
	sync.Mutex
	// contains filtered or unexported fields
}

func NewDatasetShardLocator

func NewDatasetShardLocator() *DatasetShardLocator

func (*DatasetShardLocator) GetShardLocation

func (l *DatasetShardLocator) GetShardLocation(shardName string) (pb.DataLocation, bool)

func (*DatasetShardLocator) SetShardLocation

func (l *DatasetShardLocator) SetShardLocation(name string, location pb.DataLocation)

type ReleaseTaskGroupInputs

type ReleaseTaskGroupInputs struct {
	FlowContext *flow.FlowContext
	TaskGroups  []*plan.TaskGroup
	WaitGroup   *sync.WaitGroup
}

type RemoteExecutorStatus

type RemoteExecutorStatus struct {
	Request      *msg.ControlMessage
	Allocation   *pb.Allocation
	RequestTime  time.Time
	InputLength  int
	OutputLength int
	ReadyTime    time.Time
	RunTime      time.Time
	StopTime     time.Time
}

type Scheduler

type Scheduler struct {
	sync.Mutex

	Master    string
	EventChan chan interface{}
	Market    *market.Market
	Option    *SchedulerOption

	RemoteExecutorStatuses map[uint32]*RemoteExecutorStatus
	// contains filtered or unexported fields
}

func NewScheduler

func NewScheduler(leader string, option *SchedulerOption) *Scheduler

func (*Scheduler) EventLoop

func (s *Scheduler) EventLoop()

resources are leased to driver, expires every X miniute unless renewed. 1. request resource 2. release resource

func (*Scheduler) Fetch

func (s *Scheduler) Fetch(demands []market.Demand)

Requirement is TaskGroup Object is Agent's Location

func (*Scheduler) GetShardLocation

func (s *Scheduler) GetShardLocation(shard *flow.DatasetShard) (pb.DataLocation, bool)

func (*Scheduler) Score

func (s *Scheduler) Score(r market.Requirement, bid float64, obj market.Object) float64

func (*Scheduler) SetShardLocation

func (s *Scheduler) SetShardLocation(shard *flow.DatasetShard, loc pb.DataLocation)

type SchedulerOption

type SchedulerOption struct {
	DataCenter   string
	Rack         string
	TaskMemoryMB int
	DriverHost   string
	DriverPort   int
	Module       string
}

type SubmitTaskGroup

type SubmitTaskGroup struct {
	FlowContext *flow.FlowContext
	TaskGroup   *plan.TaskGroup
	Bid         float64
	WaitGroup   *sync.WaitGroup
}

type TaskGroupStatus

type TaskGroupStatus struct {
	FlowContext *flow.FlowContext
	TaskGroup   *plan.TaskGroup
	Completed   bool
	Error       error
}