planner

package
v0.0.0-...-a18f44e Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 16, 2016 License: MIT Imports: 21 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	// States
	Starting    = dfa.State("starting")
	Running     = dfa.State("running")
	Resending   = dfa.State("resending")
	Finishing   = dfa.State("finishing")
	Exiting     = dfa.State("exiting")
	Terminating = dfa.State("terminating")
	// Letters
	Failure            = dfa.Letter("failure")
	SendFailure        = dfa.Letter("send-failure")
	SendSuccess        = dfa.Letter("send-success")
	FetchStateFailure  = dfa.Letter("fetch-state-failure")
	StoreStateFailure  = dfa.Letter("store-state-failure")
	EverybodyStarted   = dfa.Letter("everybody-started")
	EverybodyFinished  = dfa.Letter("everybody-finished")
	IndividualFinished = dfa.Letter("individual-finished")
	Exit               = dfa.Letter("exit")
)
View Source 
var (

	// BuiltIn Default Conf, used for testing but real runtime swaps this out
	//  for a real config
	GridConf = &Conf{
		GridName:    "dataux",
		EtcdServers: strings.Split("http://127.0.0.1:2379", ","),
		NatsServers: strings.Split("nats://127.0.0.1:4222", ","),
	}
)

Functions

func NewSqlActor 

func NewSqlActor(def *grid.ActorDef, conf *Conf) grid.Actor

func NextId 

func NextId() (uint64, error)

func NextIdUnsafe 

func NextIdUnsafe() uint64

func NodeName 

func NodeName(id uint64) string

func NodeName2 

func NodeName2(id1, id2 uint64) string

func RunWorkerNodes 

func RunWorkerNodes(quit chan bool, nodeCt int, r *datasource.Registry)

Types

type CmdMsg 

type CmdMsg struct {
	Cmd      string
	BodyJson u.JsonHelper
}

type Conf 

type Conf struct {
	JobMaker       JobMaker
	SchemaLoader   plan.SchemaLoader
	SupressRecover bool
	NodeCt         int
	GridName       string
	Hostname       string
	EtcdServers    []string
	NatsServers    []string
}

func (*Conf) Clone 

func (c *Conf) Clone() *Conf

type ExecutorGrid 

type ExecutorGrid struct {
	*exec.JobExecutor

	GridServer *PlannerGrid
	// contains filtered or unexported fields
}

Sql job that wraps the generic qlbridge job builder

  • contains ref to the shared GridServer which has info to distribute tasks across servers

func BuildExecutorUnPlanned 

func BuildExecutorUnPlanned(ctx *plan.Context, pg *PlannerGrid) (*ExecutorGrid, error)

Build a Sql Job which has already been planned so this is just execution runner

func BuildSqlJob 

func BuildSqlJob(ctx *plan.Context, pg *PlannerGrid) (*ExecutorGrid, error)

Build a Sql Job which may be a Grid/Distributed job

func (*ExecutorGrid) Finalize 

func (m *ExecutorGrid) Finalize(resultWriter exec.Task) error

Finalize is after the Dag of Relational-algebra tasks have been assembled 

and just before we run them.

func (*ExecutorGrid) WalkGroupBy 

func (m *ExecutorGrid) WalkGroupBy(p *plan.GroupBy) (exec.Task, error)
func (m *ExecutorGrid) WalkProjection(p *plan.Projection) (exec.Task, error) {
	u.Debugf("%p Walk Projection  sp:%+v", m, m.sp)
	return exec.NewProjection(m.Ctx, p), nil
}

func (*ExecutorGrid) WalkSelect 

func (m *ExecutorGrid) WalkSelect(p *plan.Select) (exec.Task, error)

func (*ExecutorGrid) WalkSelectPartition 

func (m *ExecutorGrid) WalkSelectPartition(p *plan.Select, part *schema.Partition) (exec.Task, error)

WalkSelectPartition is ONLY called by child-dag's, ie the remote end of a distributed 

sql query, to allow setup before walking

func (*ExecutorGrid) WalkSource 

func (m *ExecutorGrid) WalkSource(p *plan.Source) (exec.Task, error)

type Flow 

type Flow string

func NewFlow 

func NewFlow(nr uint64) Flow

func (Flow) Name 

func (f Flow) Name() string

func (Flow) NewContextualName 

func (f Flow) NewContextualName(name string) string

func (Flow) String 

func (f Flow) String() string

type JobMaker 

type JobMaker func(ctx *plan.Context) (*ExecutorGrid, error)

type PlannerGrid 

type PlannerGrid struct {
	Conf *Conf

	Grid grid.Grid
	// contains filtered or unexported fields
}

PlannerGrid that manages the sql tasks, workers

func NewServerPlanner 

func NewServerPlanner(nodeCt int, r *datasource.Registry) *PlannerGrid

func (*PlannerGrid) Run 

func (m *PlannerGrid) Run(quit chan bool) error

func (*PlannerGrid) RunSqlMaster 

func (m *PlannerGrid) RunSqlMaster(completionTask exec.TaskRunner, ns *SourceNats, flow Flow, p *plan.Select) error

Submits a Sql Select statement task for planning across multiple nodes

type SinkNats 

type SinkNats struct {
	*exec.TaskBase
	// contains filtered or unexported fields
}

SinkNats task that receives messages that optionally may have been 

hashed to be sent via nats to a nats source consumer.

taska-1 ->  hash-key -> nats-sink--> \                 / --> nats-source -->
                                      \               /
                                       --> gnatsd  -->
                                      /               \
taska-2 ->  hash-key -> nats-sink--> /                 \ --> nats-source -->

func NewSinkNats 

func NewSinkNats(ctx *plan.Context, destination string, tx grid.Sender) *SinkNats

NewSinkNats gnats sink to route messages via gnatsd

func (*SinkNats) Close 

func (m *SinkNats) Close() error

Close closes and cleanup

func (*SinkNats) CloseFinal 

func (m *SinkNats) CloseFinal() error

CloseFinal after shutdown cleanup the rest of channels

func (*SinkNats) Run 

func (m *SinkNats) Run() error

Run blocking runner

type SourceNats 

type SourceNats struct {
	*exec.TaskBase
	// contains filtered or unexported fields
}

SourceNats task that receives messages via Gnatsd, for distribution 

across multiple workers.  These messages optionally may have been
 hash routed to this node, ie partition-key routed.

 taska-1 ->  hash-nats-sink  \                        / --> nats-source -->
                              \                      /
                               --nats-route-by-key-->   --> nats-source -->
                              /                      \
 taska-2 ->  hash-nats-sink  /                        \ --> nats-source -->

func NewSourceNats 

func NewSourceNats(ctx *plan.Context, rx grid.Receiver) *SourceNats

Nats Source, the plan already provided info to the nats listener about which key/topic to listen to, Planner holds routing info not here.

func (*SourceNats) Close 

func (m *SourceNats) Close() error

Close cleans up and closes channels

func (*SourceNats) CloseFinal 

func (m *SourceNats) CloseFinal() error

CloseFinal after exit, cleanup some more

func (*SourceNats) Run 

func (m *SourceNats) Run() error

Run a blocking runner

type SqlActor 

type SqlActor struct {
	ActorCt int
	// contains filtered or unexported fields
}

Our actor for running SQL tasks in distributed grid nodes

func (*SqlActor) Act 

func (a *SqlActor) Act(g grid.Grid, exit <-chan bool) bool

func (*SqlActor) Exiting 

func (m *SqlActor) Exiting()

func (*SqlActor) Finishing 

func (m *SqlActor) Finishing() dfa.Letter

func (*SqlActor) ID 

func (a *SqlActor) ID() string

func (*SqlActor) Running 

func (m *SqlActor) Running() dfa.Letter

func (*SqlActor) Starting 

func (m *SqlActor) Starting() dfa.Letter

func (*SqlActor) String 

func (a *SqlActor) String() string

func (*SqlActor) Terminating 

func (m *SqlActor) Terminating()

type SqlState 

type SqlState struct {
	Start          time.Time
	ConsumerCounts map[string]int
	ProducerCounts map[string]int
}

State of a single actor, persisted upon stop, transfer nodes

func NewSqlState 

func NewSqlState() *SqlState

type TaskServer 

type TaskServer struct {
	Conf *Conf

	Grid grid.Grid
	// contains filtered or unexported fields
}

TaskServer accepts and performs

func NewTaskServer 

func NewTaskServer(nodeCt int, r *datasource.Registry) *TaskServer

func (*TaskServer) Run 

func (s *TaskServer) Run(quit chan bool) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.