worker

package
v1.7.7 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 6, 2018 License: Apache-2.0 Imports: 62 Imported by: 0

Documentation

Overview

Package worker is a generated protocol buffer package.

It is generated from these files:

server/worker/worker_service.proto

It has these top-level messages:

Input
CancelRequest
CancelResponse
ChunkState
Chunks

Index

Constants

View Source
const (
	// PrometheusPort is the port the aggregated metrics are served on for scraping
	PrometheusPort = 9090
)
View Source
const (
	// WorkerEtcdPrefix is the prefix in etcd that we use to store worker information.
	WorkerEtcdPrefix = "workers"
)

Variables

View Source
var (
	ErrInvalidLengthWorkerService = fmt.Errorf("proto: negative length found during unmarshaling")
	ErrIntOverflowWorkerService   = fmt.Errorf("proto: integer overflow")
)
View Source
var ChunkState_State_name = map[int32]string{
	0: "RUNNING",
	1: "COMPLETE",
	3: "FAILED",
}
View Source
var ChunkState_State_value = map[string]int32{
	"RUNNING":  0,
	"COMPLETE": 1,
	"FAILED":   3,
}

Functions

func Cancel added in v1.7.5

func Cancel(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client,
	etcdPrefix string, jobID string, dataFilter []string) error

Cancel cancels a set of datums running on workers. pipelineRcName is the name of the pipeline's RC and can be gotten with ppsutil.PipelineRcName.

func Conns added in v1.7.5

func Conns(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string) ([]*grpc.ClientConn, error)

Conns returns a slice of connections to worker servers. pipelineRcName is the name of the pipeline's RC and can be gotten with ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all clients for all workers.

func HashDatum

func HashDatum(pipelineName string, pipelineSalt string, data []*Input) string

HashDatum computes and returns the hash of datum + pipeline, with a pipeline-specific prefix.

func HashDatum15

func HashDatum15(pipelineInfo *pps.PipelineInfo, data []*Input) (string, error)

HashDatum15 computes and returns the hash of datum + pipeline for version <= 1.5.0, with a pipeline-specific prefix.

func MatchDatum

func MatchDatum(filter []string, data []*pps.InputFile) bool

MatchDatum checks if a datum matches a filter. To match each string in filter must correspond match at least 1 datum's Path or Hash. Order of filter and data is irrelevant.

func RegisterWorkerServer

func RegisterWorkerServer(s *grpc.Server, srv WorkerServer)

func Status added in v1.7.5

func Status(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string) ([]*pps.WorkerStatus, error)

Status returns the statuses of workers referenced by pipelineRcName. pipelineRcName is the name of the pipeline's RC and can be gotten with ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all clients for all workers.

Types

type APIServer

type APIServer struct {
	// contains filtered or unexported fields
}

APIServer implements the worker API

func NewAPIServer

func NewAPIServer(pachClient *client.APIClient, etcdClient *etcd.Client, etcdPrefix string, pipelineInfo *pps.PipelineInfo, workerName string, namespace string) (*APIServer, error)

NewAPIServer creates an APIServer for a given pipeline

func (*APIServer) Cancel

func (a *APIServer) Cancel(ctx context.Context, request *CancelRequest) (*CancelResponse, error)

Cancel cancels the currently running datum

func (*APIServer) DatumID added in v1.6.0

func (a *APIServer) DatumID(data []*Input) string

DatumID computes the id for a datum, this value is used in ListDatum and InspectDatum.

func (*APIServer) Status

func (a *APIServer) Status(ctx context.Context, _ *types.Empty) (*pps.WorkerStatus, error)

Status returns the status of the current worker.

type CancelRequest

type CancelRequest struct {
	JobID       string   `protobuf:"bytes,2,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
	DataFilters []string `protobuf:"bytes,1,rep,name=data_filters,json=dataFilters" json:"data_filters,omitempty"`
}

func (*CancelRequest) Descriptor

func (*CancelRequest) Descriptor() ([]byte, []int)

func (*CancelRequest) GetDataFilters

func (m *CancelRequest) GetDataFilters() []string

func (*CancelRequest) GetJobID

func (m *CancelRequest) GetJobID() string

func (*CancelRequest) Marshal

func (m *CancelRequest) Marshal() (dAtA []byte, err error)

func (*CancelRequest) MarshalTo

func (m *CancelRequest) MarshalTo(dAtA []byte) (int, error)

func (*CancelRequest) ProtoMessage

func (*CancelRequest) ProtoMessage()

func (*CancelRequest) Reset

func (m *CancelRequest) Reset()

func (*CancelRequest) Size

func (m *CancelRequest) Size() (n int)

func (*CancelRequest) String

func (m *CancelRequest) String() string

func (*CancelRequest) Unmarshal

func (m *CancelRequest) Unmarshal(dAtA []byte) error

type CancelResponse

type CancelResponse struct {
	Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"`
}

func (*CancelResponse) Descriptor

func (*CancelResponse) Descriptor() ([]byte, []int)

func (*CancelResponse) GetSuccess

func (m *CancelResponse) GetSuccess() bool

func (*CancelResponse) Marshal

func (m *CancelResponse) Marshal() (dAtA []byte, err error)

func (*CancelResponse) MarshalTo

func (m *CancelResponse) MarshalTo(dAtA []byte) (int, error)

func (*CancelResponse) ProtoMessage

func (*CancelResponse) ProtoMessage()

func (*CancelResponse) Reset

func (m *CancelResponse) Reset()

func (*CancelResponse) Size

func (m *CancelResponse) Size() (n int)

func (*CancelResponse) String

func (m *CancelResponse) String() string

func (*CancelResponse) Unmarshal

func (m *CancelResponse) Unmarshal(dAtA []byte) error

type ChunkState added in v1.6.6

type ChunkState struct {
	State   ChunkState_State `protobuf:"varint,1,opt,name=state,proto3,enum=worker.ChunkState_State" json:"state,omitempty"`
	DatumID string           `protobuf:"bytes,2,opt,name=datum_id,json=datumId,proto3" json:"datum_id,omitempty"`
}

func (*ChunkState) Descriptor added in v1.6.6

func (*ChunkState) Descriptor() ([]byte, []int)

func (*ChunkState) GetDatumID added in v1.6.6

func (m *ChunkState) GetDatumID() string

func (*ChunkState) GetState added in v1.6.6

func (m *ChunkState) GetState() ChunkState_State

func (*ChunkState) Marshal added in v1.6.6

func (m *ChunkState) Marshal() (dAtA []byte, err error)

func (*ChunkState) MarshalTo added in v1.6.6

func (m *ChunkState) MarshalTo(dAtA []byte) (int, error)

func (*ChunkState) ProtoMessage added in v1.6.6

func (*ChunkState) ProtoMessage()

func (*ChunkState) Reset added in v1.6.6

func (m *ChunkState) Reset()

func (*ChunkState) Size added in v1.6.6

func (m *ChunkState) Size() (n int)

func (*ChunkState) String added in v1.6.6

func (m *ChunkState) String() string

func (*ChunkState) Unmarshal added in v1.6.6

func (m *ChunkState) Unmarshal(dAtA []byte) error

type ChunkState_State added in v1.6.6

type ChunkState_State int32
const (
	ChunkState_RUNNING  ChunkState_State = 0
	ChunkState_COMPLETE ChunkState_State = 1
	ChunkState_FAILED   ChunkState_State = 3
)

func (ChunkState_State) EnumDescriptor added in v1.6.6

func (ChunkState_State) EnumDescriptor() ([]byte, []int)

func (ChunkState_State) String added in v1.6.6

func (x ChunkState_State) String() string

type Chunks added in v1.6.6

type Chunks struct {
	Chunks []int64 `protobuf:"varint,1,rep,packed,name=chunks" json:"chunks,omitempty"`
}

func (*Chunks) Descriptor added in v1.6.6

func (*Chunks) Descriptor() ([]byte, []int)

func (*Chunks) GetChunks added in v1.6.6

func (m *Chunks) GetChunks() []int64

func (*Chunks) Marshal added in v1.6.6

func (m *Chunks) Marshal() (dAtA []byte, err error)

func (*Chunks) MarshalTo added in v1.6.6

func (m *Chunks) MarshalTo(dAtA []byte) (int, error)

func (*Chunks) ProtoMessage added in v1.6.6

func (*Chunks) ProtoMessage()

func (*Chunks) Reset added in v1.6.6

func (m *Chunks) Reset()

func (*Chunks) Size added in v1.6.6

func (m *Chunks) Size() (n int)

func (*Chunks) String added in v1.6.6

func (m *Chunks) String() string

func (*Chunks) Unmarshal added in v1.6.6

func (m *Chunks) Unmarshal(dAtA []byte) error

type Client added in v1.7.5

type Client struct {
	WorkerClient
	debug.DebugClient
}

Client combines the WorkerAPI and the DebugAPI into a single client.

func Clients added in v1.7.5

func Clients(ctx context.Context, pipelineRcName string, etcdClient *etcd.Client, etcdPrefix string) ([]Client, error)

Clients returns a slice of worker clients for a pipeline. pipelineRcName is the name of the pipeline's RC and can be gotten with ppsutil.PipelineRcName. You can also pass "" for pipelineRcName to get all clients for all workers.

type DatumFactory added in v1.5.3

type DatumFactory interface {
	Len() int
	Datum(i int) []*Input
}

DatumFactory is an interface which allows you to iterate through the datums for a job.

func NewDatumFactory added in v1.5.3

func NewDatumFactory(pachClient *client.APIClient, input *pps.Input) (DatumFactory, error)

NewDatumFactory creates a datumFactory for an input.

type Input

type Input struct {
	FileInfo     *pfs.FileInfo `protobuf:"bytes,1,opt,name=file_info,json=fileInfo" json:"file_info,omitempty"`
	ParentCommit *pfs.Commit   `protobuf:"bytes,5,opt,name=parent_commit,json=parentCommit" json:"parent_commit,omitempty"`
	Name         string        `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
	Lazy         bool          `protobuf:"varint,3,opt,name=lazy,proto3" json:"lazy,omitempty"`
	Branch       string        `protobuf:"bytes,4,opt,name=branch,proto3" json:"branch,omitempty"`
	GitURL       string        `protobuf:"bytes,6,opt,name=git_url,json=gitUrl,proto3" json:"git_url,omitempty"`
	EmptyFiles   bool          `protobuf:"varint,7,opt,name=empty_files,json=emptyFiles,proto3" json:"empty_files,omitempty"`
}

func (*Input) Descriptor

func (*Input) Descriptor() ([]byte, []int)

func (*Input) GetBranch

func (m *Input) GetBranch() string

func (*Input) GetEmptyFiles added in v1.6.8

func (m *Input) GetEmptyFiles() bool

func (*Input) GetFileInfo

func (m *Input) GetFileInfo() *pfs.FileInfo

func (*Input) GetGitURL added in v1.6.4

func (m *Input) GetGitURL() string

func (*Input) GetLazy

func (m *Input) GetLazy() bool

func (*Input) GetName

func (m *Input) GetName() string

func (*Input) GetParentCommit

func (m *Input) GetParentCommit() *pfs.Commit

func (*Input) Marshal

func (m *Input) Marshal() (dAtA []byte, err error)

func (*Input) MarshalTo

func (m *Input) MarshalTo(dAtA []byte) (int, error)

func (*Input) ProtoMessage

func (*Input) ProtoMessage()

func (*Input) Reset

func (m *Input) Reset()

func (*Input) Size

func (m *Input) Size() (n int)

func (*Input) String

func (m *Input) String() string

func (*Input) Unmarshal

func (m *Input) Unmarshal(dAtA []byte) error

type WorkerClient

type WorkerClient interface {
	Status(ctx context.Context, in *google_protobuf.Empty, opts ...grpc.CallOption) (*pps.WorkerStatus, error)
	Cancel(ctx context.Context, in *CancelRequest, opts ...grpc.CallOption) (*CancelResponse, error)
}

func NewWorkerClient

func NewWorkerClient(cc *grpc.ClientConn) WorkerClient

type WorkerServer

type WorkerServer interface {
	Status(context.Context, *google_protobuf.Empty) (*pps.WorkerStatus, error)
	Cancel(context.Context, *CancelRequest) (*CancelResponse, error)
}

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL