stage

Documentation

Index

• Constants
• func CmdAllStoresCompleted() loop.Cmd
• func CmdMergeNotReady(nextUnit Unit, reason string) loop.Cmd
• type Kind
• type MsgAllStoresCompleted
• type MsgMergeFailed
• type MsgMergeFinished
• type MsgMergeNotReady
• type Result
• type Stage
  • func NewStage(idx int, kind Kind, segmenter *block.Segmenter, ...) *Stage
• type Stages
  • func NewStages(ctx context.Context, execGraph *exec.Graph, reqPlan *plan.RequestPlan, ...) (out *Stages)
  • func (s *Stages) AllStoresCompleted() bool
  • func (s *Stages) CmdStartMerge() loop.Cmd
  • func (s *Stages) CmdTryMerge(stageIdx int) loop.Cmd
  • func (s *Stages) FetchStoresState(ctx context.Context, segmenter *block.Segmenter, ...) error
  • func (s *Stages) FinalStoreMap(exclusiveEndBlock uint64) (store.Map, error)
  • func (s *Stages) LastStageCompleted() bool
  • func (s *Stages) MarkJobSuccess(u Unit) (shadowedUnits []Unit)
  • func (s *Stages) MarkSegmentMerging(u Unit)
  • func (s *Stages) MarkSegmentPartialPresent(u Unit)
  • func (s *Stages) MarkSegmentPending(u Unit)
  • func (s *Stages) MergeCompleted(mergeUnit Unit)
  • func (s *Stages) MoveSegmentCompletedForward(stageIdx int)
  • func (s *Stages) NextJob() (Unit, *block.Range)
  • func (s *Stages) OutputModuleIsIndex() bool
  • func (s *Stages) StageModules(stage int) (out []string)
  • func (s *Stages) StatesString() string
  • func (s *Stages) UpdateStats()
  • func (s *Stages) WaitAsyncWork() error
• type StoreModuleState
  • func NewModuleState(logger *zap.Logger, name string, segmenter *block.Segmenter, ...) *StoreModuleState
  • func (s *StoreModuleState) Name() string
• type Unit
  • func (u Unit) MarshalLogObject(enc zapcore.ObjectEncoder) error
• type UnitState
  • func (s UnitState) String() string

Constants

const (
	KindMap = Kind(iota)
	KindStore
)

Functions

func CmdAllStoresCompleted

func CmdAllStoresCompleted() loop.Cmd

func CmdMergeNotReady

func CmdMergeNotReady(nextUnit Unit, reason string) loop.Cmd

Types

type Kind

type Kind int

type MsgAllStoresCompleted

type MsgAllStoresCompleted struct {
	Unit
}

This means that this single Store has completed its full sync, up to the target block

type MsgMergeFailed

type MsgMergeFailed struct {
	Unit
	Error error
}

type MsgMergeFinished

type MsgMergeFinished struct {
	Unit

} // A single partial store was successfully merged into the full store.

type MsgMergeNotReady

type MsgMergeNotReady struct {
	Reason   string
	NextUnit Unit
}

type Result

type Result struct {
	// contains filtered or unexported fields
}

type Stage

type Stage struct {
	// contains filtered or unexported fields
}

func NewStage

func NewStage(idx int, kind Kind, segmenter *block.Segmenter, moduleStates []*StoreModuleState, allExecutedModules []string) *Stage

type Stages

type Stages struct {
	// contains filtered or unexported fields
}

func NewStages

func NewStages(
	ctx context.Context,
	execGraph *exec.Graph,
	reqPlan *plan.RequestPlan,
	storeConfigs store.ConfigMap,
) (out *Stages)

func (*Stages) AllStoresCompleted

func (s *Stages) AllStoresCompleted() bool

func (*Stages) CmdStartMerge

func (s *Stages) CmdStartMerge() loop.Cmd

func (*Stages) CmdTryMerge

func (s *Stages) CmdTryMerge(stageIdx int) loop.Cmd

func (*Stages) FetchStoresState

func (s *Stages) FetchStoresState(
	ctx context.Context,
	segmenter *block.Segmenter,
	storeConfigMap store.ConfigMap,
	execoutConfigs *execout.Configs,
) error

func (*Stages) FinalStoreMap

func (s *Stages) FinalStoreMap(exclusiveEndBlock uint64) (store.Map, error)

func (*Stages) LastStageCompleted

func (s *Stages) LastStageCompleted() bool

func (*Stages) MarkJobSuccess

func (s *Stages) MarkJobSuccess(u Unit) (shadowedUnits []Unit)

func (*Stages) MarkSegmentMerging

func (s *Stages) MarkSegmentMerging(u Unit)

func (*Stages) MarkSegmentPartialPresent

func (s *Stages) MarkSegmentPartialPresent(u Unit)

func (*Stages) MarkSegmentPending

func (s *Stages) MarkSegmentPending(u Unit)

func (*Stages) MergeCompleted

func (s *Stages) MergeCompleted(mergeUnit Unit)

func (*Stages) MoveSegmentCompletedForward

func (s *Stages) MoveSegmentCompletedForward(stageIdx int)

func (*Stages) NextJob

func (s *Stages) NextJob() (Unit, *block.Range)

func (*Stages) OutputModuleIsIndex

func (s *Stages) OutputModuleIsIndex() bool

func (*Stages) StageModules

func (s *Stages) StageModules(stage int) (out []string)

func (*Stages) StatesString

func (s *Stages) StatesString() string

func (*Stages) UpdateStats

func (s *Stages) UpdateStats()

UpdateStats is gated to be called at most once per second. It runs the first time it is called.

func (*Stages) WaitAsyncWork

func (s *Stages) WaitAsyncWork() error

type StoreModuleState

type StoreModuleState struct {
	// contains filtered or unexported fields
}

An individual module's progress towards synchronizing its `store`

func NewModuleState

func NewModuleState(logger *zap.Logger, name string, segmenter *block.Segmenter, storeConfig *store.Config) *StoreModuleState

func (*StoreModuleState) Name

func (s *StoreModuleState) Name() string

type Unit

type Unit struct {
	Segment int
	Stage   int
}

Unit can be used as a key, and points to the respective indexes of Stages.getState(unit)

func (Unit) MarshalLogObject

func (u Unit) MarshalLogObject(enc zapcore.ObjectEncoder) error

type UnitState

type UnitState int

const (
	UnitPending UnitState = iota // The job needs to be scheduled, no complete store exists at the end of its Range, nor any partial store for the end of this segment.
	UnitPartialPresent
	UnitScheduled // Means the job was scheduled for execution
	UnitMerging   // A partial is being merged
	UnitShadowed  // will not be run directly, its outputs are created by the last stage of this segment
	UnitCompleted // End state. A store has been snapshot for this segment, and we have gone over in the per-request squasher
	UnitNoOp      // State given to a unit that does not need scheduling. Mostly for map segments where we know in advance we won't consume the output.
)

func (UnitState) String

func (s UnitState) String() string