task

package
v1.1.27 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 17, 2022 License: Apache-2.0 Imports: 48 Imported by: 0

Documentation

Index

Constants

View Source 
const DefaultMaxAttempts = 1
View Source 
const IDMaxLength = 50

Variables

View Source 
var NoBarrierTransition = BarrierTransition{BarrierClockTick: 0}

Functions

func ComputePreviousCheckpointPath added in v0.16.0 

func ComputePreviousCheckpointPath(ctx context.Context, length int, nCtx handler.NodeExecutionContext, currentNodeUniqueID v1alpha1.NodeID, currentAttempt uint32) (storage.DataReference, error)

ComputePreviousCheckpointPath returns the checkpoint path for the previous attempt, if this is the first attempt then returns an empty path

func ComputeRawOutputPrefix added in v0.16.0 

func ComputeRawOutputPrefix(ctx context.Context, length int, nCtx handler.NodeExecutionContext, currentNodeUniqueID v1alpha1.NodeID, currentAttempt uint32) (io.RawOutputPaths, string, error)

ComputeRawOutputPrefix constructs the output directory, where raw outputs of a task can be stored by the task. FlytePropeller may not have access to this location and can be passed in per execution. the function also returns the uniqueID generated

func GetTaskExecutionIdentifier added in v0.1.13 

func GetTaskExecutionIdentifier(nCtx handler.NodeExecutionContext) *core.TaskExecutionIdentifier

func ToTaskEventPhase added in v0.1.13 

func ToTaskEventPhase(p pluginCore.Phase) core.TaskExecution_Phase

func ToTaskExecutionEvent added in v0.1.13 

func ToTaskExecutionEvent(input ToTaskExecutionEventInputs) (*event.TaskExecutionEvent, error)

func ToTransitionType added in v0.1.13 

func ToTransitionType(ttype pluginCore.TransitionType) handler.TransitionType

func WranglePluginsAndGenerateFinalList added in v0.1.13 

func WranglePluginsAndGenerateFinalList(ctx context.Context, cfg *config.TaskPluginConfig, pr PluginRegistryIface) (enabledPlugins []core.PluginEntry, defaultForTaskTypes map[pluginID][]taskType, err error)

Types

type BarrierKey added in v0.1.13 

type BarrierKey = string

type BarrierTransition added in v0.1.13 

type BarrierTransition struct {
	BarrierClockTick uint32
	CallLog          PluginCallLog
}

type CacheContents added in v0.10.5 

type CacheContents struct {
	WorkflowCRD      *v1alpha1.FlyteWorkflow
	CompiledWorkflow *core.CompiledWorkflowClosure
}

type CodecVersion added in v0.1.13 

type CodecVersion uint8
const (
	GobCodecVersion CodecVersion = iota
)

type FutureFileReader added in v0.1.13 

type FutureFileReader struct {
	RemoteFileWorkflowStore
	// contains filtered or unexported fields
}

func NewRemoteFutureFileReader added in v0.1.13 

func NewRemoteFutureFileReader(ctx context.Context, dataDir storage.DataReference, store *storage.DataStore) (FutureFileReader, error)

func (FutureFileReader) Cache added in v0.1.13 

func (f FutureFileReader) Cache(ctx context.Context, wf *v1alpha1.FlyteWorkflow, workflowClosure *core.CompiledWorkflowClosure) error

func (FutureFileReader) CacheExists added in v0.1.13 

func (f FutureFileReader) CacheExists(ctx context.Context) (bool, error)

func (FutureFileReader) Exists added in v0.1.13 

func (f FutureFileReader) Exists(ctx context.Context) (bool, error)

func (FutureFileReader) Read added in v0.1.13 

func (f FutureFileReader) Read(ctx context.Context) (*core.DynamicJobSpec, error)

func (FutureFileReader) RetrieveCache added in v0.1.13 

func (f FutureFileReader) RetrieveCache(ctx context.Context) (CacheContents, error)

type Handler added in v0.1.13 

type Handler struct {
	// contains filtered or unexported fields
}

func New 

func New(ctx context.Context, kubeClient executors.Client, client catalog.Client, eventConfig *controllerConfig.EventConfig, clusterID string, scope promutils.Scope) (*Handler, error)

func (Handler) Abort added in v0.1.13 

func (t Handler) Abort(ctx context.Context, nCtx handler.NodeExecutionContext, reason string) error

func (*Handler) CheckCatalogCache added in v0.1.13 

func (t *Handler) CheckCatalogCache(ctx context.Context, tr pluginCore.TaskReader, inputReader io.InputReader, outputWriter io.OutputWriter) (catalog.Entry, error)

func (Handler) Finalize added in v0.1.13 

func (t Handler) Finalize(ctx context.Context, nCtx handler.NodeExecutionContext) error

func (*Handler) FinalizeRequired added in v0.1.13 

func (t *Handler) FinalizeRequired() bool

func (*Handler) GetOrExtendCatalogReservation added in v0.16.1 

func (t *Handler) GetOrExtendCatalogReservation(ctx context.Context, ownerID string, heartbeatInterval time.Duration, tr pluginCore.TaskReader, inputReader io.InputReader) (catalog.ReservationEntry, error)

GetOrExtendCatalogReservation attempts to acquire an artifact reservation if the task is cachable and cache serializable. If the reservation already exists for this owner, the reservation is extended.

func (Handler) Handle added in v0.1.13 

func (t Handler) Handle(ctx context.Context, nCtx handler.NodeExecutionContext) (handler.Transition, error)

func (*Handler) ReleaseCatalogReservation added in v0.16.1 

func (t *Handler) ReleaseCatalogReservation(ctx context.Context, ownerID string, tr pluginCore.TaskReader, inputReader io.InputReader) (catalog.ReservationEntry, error)

ReleaseCatalogReservation attempts to release an artifact reservation if the task is cachable and cache serializable. If the reservation does not exist for this owner (e.x. it never existed or has been acquired by another owner) this call is still successful.

func (Handler) ResolvePlugin added in v0.1.13 

func (t Handler) ResolvePlugin(ctx context.Context, ttype string, executionConfig v1alpha1.ExecutionConfig) (pluginCore.Plugin, error)

func (*Handler) Setup added in v0.1.13 

func (t *Handler) Setup(ctx context.Context, sCtx handler.SetupContext) error

func (*Handler) ValidateOutputAndCacheAdd added in v0.1.13 

func (t *Handler) ValidateOutputAndCacheAdd(ctx context.Context, nodeID v1alpha1.NodeID, i io.InputReader,
	r io.OutputReader, outputCommitter io.OutputWriter, executionConfig v1alpha1.ExecutionConfig,
	tr ioutils.SimpleTaskReader, m catalog.Metadata) (catalog.Status, *io.ExecutionError, error)

type MetricKey added in v0.7.0 

type MetricKey = string

type PluginCallLog added in v0.1.13 

type PluginCallLog struct {
	PluginTransition *pluginRequestedTransition
}

type PluginRegistryIface added in v0.1.13 

type PluginRegistryIface interface {
	GetCorePlugins() []pluginCore.PluginEntry
	GetK8sPlugins() []pluginK8s.PluginEntry
}

The plugin interface available especially for testing.

type RemoteFileWorkflowStore added in v0.1.13 

type RemoteFileWorkflowStore struct {
	// contains filtered or unexported fields
}

func NewRemoteWorkflowStore added in v0.1.13 

func NewRemoteWorkflowStore(store *storage.DataStore) RemoteFileWorkflowStore

func (RemoteFileWorkflowStore) Exists added in v0.1.13 

func (r RemoteFileWorkflowStore) Exists(ctx context.Context, path storage.DataReference) (bool, error)

func (RemoteFileWorkflowStore) GetCompiledWorkflow added in v0.10.5 

func (r RemoteFileWorkflowStore) GetCompiledWorkflow(ctx context.Context, source storage.DataReference) (*core.CompiledWorkflowClosure, error)

func (RemoteFileWorkflowStore) GetWorkflowCRD added in v0.10.5 

func (r RemoteFileWorkflowStore) GetWorkflowCRD(ctx context.Context, source storage.DataReference) (*v1alpha1.FlyteWorkflow, error)

func (RemoteFileWorkflowStore) PutCompiledFlyteWorkflow added in v0.10.5 

func (r RemoteFileWorkflowStore) PutCompiledFlyteWorkflow(ctx context.Context, workflow *core.CompiledWorkflowClosure, target storage.DataReference) error

func (RemoteFileWorkflowStore) PutFlyteWorkflowCRD added in v0.10.5 

func (r RemoteFileWorkflowStore) PutFlyteWorkflowCRD(ctx context.Context, wf *v1alpha1.FlyteWorkflow, target storage.DataReference) error

type ToTaskExecutionEventInputs added in v0.7.9 

type ToTaskExecutionEventInputs struct {
	TaskExecContext       pluginCore.TaskExecutionContext
	InputReader           io.InputFilePaths
	OutputWriter          io.OutputFilePaths
	Info                  pluginCore.PhaseInfo
	NodeExecutionMetadata handler.NodeExecutionMetadata
	ExecContext           executors.ExecutionContext
	TaskType              string
	PluginID              string
	ResourcePoolInfo      []*event.ResourcePoolInfo
	ClusterID             string
}

Source Files

View all Source files

Directories

Path Synopsis
datacatalog
config
mocks

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.