worker

Documentation

Overview

Package worker is a generated protocol buffer package.

It is generated from these files:

server/pkg/worker/worker_service.proto

It has these top-level messages:

ProcessRequest
ProcessResponse
CancelRequest
CancelResponse

Index

• func MatchDatum(filter []string, data []*pps.Datum) bool
• func RegisterWorkerServer(s *grpc.Server, srv WorkerServer)
• type APIServer
  • func NewJobAPIServer(pachClient *client.APIClient, jobInfo *pps.JobInfo, workerName string) *APIServer
  • func NewPipelineAPIServer(pachClient *client.APIClient, pipelineInfo *pps.PipelineInfo, ...) *APIServer
  • func (a *APIServer) Cancel(ctx context.Context, request *CancelRequest) (*CancelResponse, error)
  • func (a *APIServer) HashDatum(data []*pfs.FileInfo) (string, error)
  • func (a *APIServer) Process(ctx context.Context, req *ProcessRequest) (resp *ProcessResponse, retErr error)
  • func (a *APIServer) Status(ctx context.Context, _ *types.Empty) (*pps.WorkerStatus, error)
• type CancelRequest
  • func (*CancelRequest) Descriptor() ([]byte, []int)
  • func (m *CancelRequest) GetDataFilters() []string
  • func (m *CancelRequest) GetJobID() string
  • func (*CancelRequest) ProtoMessage()
  • func (m *CancelRequest) Reset()
  • func (m *CancelRequest) String() string
• type CancelResponse
  • func (*CancelResponse) Descriptor() ([]byte, []int)
  • func (m *CancelResponse) GetSuccess() bool
  • func (*CancelResponse) ProtoMessage()
  • func (m *CancelResponse) Reset()
  • func (m *CancelResponse) String() string
• type Input
• type ProcessRequest
  • func (*ProcessRequest) Descriptor() ([]byte, []int)
  • func (m *ProcessRequest) GetData() []*pfs.FileInfo
  • func (m *ProcessRequest) GetJobID() string
  • func (*ProcessRequest) ProtoMessage()
  • func (m *ProcessRequest) Reset()
  • func (m *ProcessRequest) String() string
• type ProcessResponse
  • func (*ProcessResponse) Descriptor() ([]byte, []int)
  • func (m *ProcessResponse) GetFailed() bool
  • func (m *ProcessResponse) GetTag() *pfs.Tag
  • func (*ProcessResponse) ProtoMessage()
  • func (m *ProcessResponse) Reset()
  • func (m *ProcessResponse) String() string
• type WorkerClient
  • func NewWorkerClient(cc *grpc.ClientConn) WorkerClient
• type WorkerServer

Functions

func MatchDatum

func MatchDatum(filter []string, data []*pps.Datum) bool

MatchDatum checks if a datum matches a filter. To match each string in filter must correspond match at least 1 datum's Path or Hash. Order of filter and data is irrelevant.

func RegisterWorkerServer

func RegisterWorkerServer(s *grpc.Server, srv WorkerServer)

Types

type APIServer

type APIServer struct {
	// contains filtered or unexported fields
}

APIServer implements the worker API

func NewJobAPIServer

func NewJobAPIServer(pachClient *client.APIClient, jobInfo *pps.JobInfo, workerName string) *APIServer

NewJobAPIServer creates an APIServer for a given pipeline

func NewPipelineAPIServer

func NewPipelineAPIServer(pachClient *client.APIClient, pipelineInfo *pps.PipelineInfo, workerName string) *APIServer

NewPipelineAPIServer creates an APIServer for a given pipeline

func (*APIServer) Cancel

func (a *APIServer) Cancel(ctx context.Context, request *CancelRequest) (*CancelResponse, error)

Cancel cancels the currently running datum

func (*APIServer) HashDatum

func (a *APIServer) HashDatum(data []*pfs.FileInfo) (string, error)

HashDatum computes and returns the hash of a datum + pipeline.

func (*APIServer) Process

func (a *APIServer) Process(ctx context.Context, req *ProcessRequest) (resp *ProcessResponse, retErr error)

Process processes a datum.

func (*APIServer) Status

func (a *APIServer) Status(ctx context.Context, _ *types.Empty) (*pps.WorkerStatus, error)

Status returns the status of the current worker.

type CancelRequest

type CancelRequest struct {
	JobID       string   `protobuf:"bytes,2,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
	DataFilters []string `protobuf:"bytes,1,rep,name=data_filters,json=dataFilters" json:"data_filters,omitempty"`
}

func (*CancelRequest) Descriptor

func (*CancelRequest) Descriptor() ([]byte, []int)

func (*CancelRequest) GetDataFilters

func (m *CancelRequest) GetDataFilters() []string

func (*CancelRequest) GetJobID

func (m *CancelRequest) GetJobID() string

func (*CancelRequest) ProtoMessage

func (*CancelRequest) ProtoMessage()

func (*CancelRequest) Reset

func (m *CancelRequest) Reset()

func (*CancelRequest) String

func (m *CancelRequest) String() string

type CancelResponse

type CancelResponse struct {
	Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
}

func (*CancelResponse) Descriptor

func (*CancelResponse) Descriptor() ([]byte, []int)

func (*CancelResponse) GetSuccess

func (m *CancelResponse) GetSuccess() bool

func (*CancelResponse) ProtoMessage

func (*CancelResponse) ProtoMessage()

func (*CancelResponse) Reset

func (m *CancelResponse) Reset()

func (*CancelResponse) String

func (m *CancelResponse) String() string

type Input

type Input struct {
	Name string
	Lazy bool
}

Input is a generic input object that can either be a pipeline input or a job input. It only defines the attributes that the worker cares about.

type ProcessRequest

type ProcessRequest struct {
	// ID of the job for which we're processing 'data'. This is attached to logs
	// generated while processing 'data', so that they can be searched.
	JobID string `protobuf:"bytes,2,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
	// The datum to process
	Data []*pfs.FileInfo `protobuf:"bytes,1,rep,name=data" json:"data,omitempty"`
}

func (*ProcessRequest) Descriptor

func (*ProcessRequest) Descriptor() ([]byte, []int)

func (*ProcessRequest) GetData

func (m *ProcessRequest) GetData() []*pfs.FileInfo

func (*ProcessRequest) GetJobID

func (m *ProcessRequest) GetJobID() string

func (*ProcessRequest) ProtoMessage

func (*ProcessRequest) ProtoMessage()

func (*ProcessRequest) Reset

func (m *ProcessRequest) Reset()

func (*ProcessRequest) String

func (m *ProcessRequest) String() string

type ProcessResponse

type ProcessResponse struct {
	Tag *pfs.Tag `protobuf:"bytes,1,opt,name=tag" json:"tag,omitempty"`
	// If true, the user program has errored
	Failed bool `protobuf:"varint,2,opt,name=failed,proto3" json:"failed,omitempty"`
}

ProcessResponse contains a tag, only if the processing was successful.

func (*ProcessResponse) Descriptor

func (*ProcessResponse) Descriptor() ([]byte, []int)

func (*ProcessResponse) GetFailed

func (m *ProcessResponse) GetFailed() bool

func (*ProcessResponse) GetTag

func (m *ProcessResponse) GetTag() *pfs.Tag

func (*ProcessResponse) ProtoMessage

func (*ProcessResponse) ProtoMessage()

func (*ProcessResponse) Reset

func (m *ProcessResponse) Reset()

func (*ProcessResponse) String

func (m *ProcessResponse) String() string

type WorkerClient

type WorkerClient interface {
	Process(ctx context.Context, in *ProcessRequest, opts ...grpc.CallOption) (*ProcessResponse, error)
	Status(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*pps.WorkerStatus, error)
	Cancel(ctx context.Context, in *CancelRequest, opts ...grpc.CallOption) (*CancelResponse, error)
}

func NewWorkerClient

func NewWorkerClient(cc *grpc.ClientConn) WorkerClient

type WorkerServer

type WorkerServer interface {
	Process(context.Context, *ProcessRequest) (*ProcessResponse, error)
	Status(context.Context, *google_protobuf.Empty) (*pps.WorkerStatus, error)
	Cancel(context.Context, *CancelRequest) (*CancelResponse, error)
}