Documentation ¶
Overview ¶
Package dataflowdatav1beta3 provides Dataflow type definitions for CloudEvent data payloads.
Supported CloudEvent Types ¶
- google.cloud.dataflow.job.v1beta3.statusChanged
Index ¶
- Variables
- type AutoscalingAlgorithm
- func (AutoscalingAlgorithm) Descriptor() protoreflect.EnumDescriptor
- func (x AutoscalingAlgorithm) Enum() *AutoscalingAlgorithm
- func (AutoscalingAlgorithm) EnumDescriptor() ([]byte, []int)deprecated
- func (x AutoscalingAlgorithm) Number() protoreflect.EnumNumber
- func (x AutoscalingAlgorithm) String() string
- func (AutoscalingAlgorithm) Type() protoreflect.EnumType
- type AutoscalingSettings
- func (*AutoscalingSettings) Descriptor() ([]byte, []int)deprecated
- func (x *AutoscalingSettings) GetAlgorithm() AutoscalingAlgorithm
- func (x *AutoscalingSettings) GetMaxNumWorkers() int32
- func (*AutoscalingSettings) ProtoMessage()
- func (x *AutoscalingSettings) ProtoReflect() protoreflect.Message
- func (x *AutoscalingSettings) Reset()
- func (x *AutoscalingSettings) String() string
- type BigQueryIODetails
- func (*BigQueryIODetails) Descriptor() ([]byte, []int)deprecated
- func (x *BigQueryIODetails) GetDataset() string
- func (x *BigQueryIODetails) GetProjectId() string
- func (x *BigQueryIODetails) GetQuery() string
- func (x *BigQueryIODetails) GetTable() string
- func (*BigQueryIODetails) ProtoMessage()
- func (x *BigQueryIODetails) ProtoReflect() protoreflect.Message
- func (x *BigQueryIODetails) Reset()
- func (x *BigQueryIODetails) String() string
- type BigTableIODetails
- func (*BigTableIODetails) Descriptor() ([]byte, []int)deprecated
- func (x *BigTableIODetails) GetInstanceId() string
- func (x *BigTableIODetails) GetProjectId() string
- func (x *BigTableIODetails) GetTableId() string
- func (*BigTableIODetails) ProtoMessage()
- func (x *BigTableIODetails) ProtoReflect() protoreflect.Message
- func (x *BigTableIODetails) Reset()
- func (x *BigTableIODetails) String() string
- type DatastoreIODetails
- func (*DatastoreIODetails) Descriptor() ([]byte, []int)deprecated
- func (x *DatastoreIODetails) GetNamespace() string
- func (x *DatastoreIODetails) GetProjectId() string
- func (*DatastoreIODetails) ProtoMessage()
- func (x *DatastoreIODetails) ProtoReflect() protoreflect.Message
- func (x *DatastoreIODetails) Reset()
- func (x *DatastoreIODetails) String() string
- type DebugOptions
- type DefaultPackageSet
- func (DefaultPackageSet) Descriptor() protoreflect.EnumDescriptor
- func (x DefaultPackageSet) Enum() *DefaultPackageSet
- func (DefaultPackageSet) EnumDescriptor() ([]byte, []int)deprecated
- func (x DefaultPackageSet) Number() protoreflect.EnumNumber
- func (x DefaultPackageSet) String() string
- func (DefaultPackageSet) Type() protoreflect.EnumType
- type Environment
- func (*Environment) Descriptor() ([]byte, []int)deprecated
- func (x *Environment) GetClusterManagerApiService() string
- func (x *Environment) GetDataset() string
- func (x *Environment) GetDebugOptions() *DebugOptions
- func (x *Environment) GetExperiments() []string
- func (x *Environment) GetFlexResourceSchedulingGoal() FlexResourceSchedulingGoal
- func (x *Environment) GetSdkPipelineOptions() *structpb.Struct
- func (x *Environment) GetServiceAccountEmail() string
- func (x *Environment) GetServiceKmsKeyName() string
- func (x *Environment) GetServiceOptions() []string
- func (x *Environment) GetShuffleMode() ShuffleMode
- func (x *Environment) GetTempStoragePrefix() string
- func (x *Environment) GetUserAgent() *structpb.Struct
- func (x *Environment) GetVersion() *structpb.Struct
- func (x *Environment) GetWorkerPools() []*WorkerPool
- func (x *Environment) GetWorkerRegion() string
- func (x *Environment) GetWorkerZone() string
- func (*Environment) ProtoMessage()
- func (x *Environment) ProtoReflect() protoreflect.Message
- func (x *Environment) Reset()
- func (x *Environment) String() string
- type ExecutionStageState
- func (*ExecutionStageState) Descriptor() ([]byte, []int)deprecated
- func (x *ExecutionStageState) GetCurrentStateTime() *timestamppb.Timestamp
- func (x *ExecutionStageState) GetExecutionStageName() string
- func (x *ExecutionStageState) GetExecutionStageState() JobState
- func (*ExecutionStageState) ProtoMessage()
- func (x *ExecutionStageState) ProtoReflect() protoreflect.Message
- func (x *ExecutionStageState) Reset()
- func (x *ExecutionStageState) String() string
- type FileIODetails
- type FlexResourceSchedulingGoal
- func (FlexResourceSchedulingGoal) Descriptor() protoreflect.EnumDescriptor
- func (x FlexResourceSchedulingGoal) Enum() *FlexResourceSchedulingGoal
- func (FlexResourceSchedulingGoal) EnumDescriptor() ([]byte, []int)deprecated
- func (x FlexResourceSchedulingGoal) Number() protoreflect.EnumNumber
- func (x FlexResourceSchedulingGoal) String() string
- func (FlexResourceSchedulingGoal) Type() protoreflect.EnumType
- type Job
- func (*Job) Descriptor() ([]byte, []int)deprecated
- func (x *Job) GetClientRequestId() string
- func (x *Job) GetCreateTime() *timestamppb.Timestamp
- func (x *Job) GetCreatedFromSnapshotId() string
- func (x *Job) GetCurrentState() JobState
- func (x *Job) GetCurrentStateTime() *timestamppb.Timestamp
- func (x *Job) GetEnvironment() *Environment
- func (x *Job) GetExecutionInfo() *JobExecutionInfo
- func (x *Job) GetId() string
- func (x *Job) GetJobMetadata() *JobMetadata
- func (x *Job) GetLabels() map[string]string
- func (x *Job) GetLocation() string
- func (x *Job) GetName() string
- func (x *Job) GetProjectId() string
- func (x *Job) GetReplaceJobId() string
- func (x *Job) GetReplacedByJobId() string
- func (x *Job) GetRequestedState() JobState
- func (x *Job) GetSatisfiesPzs() bool
- func (x *Job) GetStageStates() []*ExecutionStageState
- func (x *Job) GetStartTime() *timestamppb.Timestamp
- func (x *Job) GetStepsLocation() string
- func (x *Job) GetTempFiles() []string
- func (x *Job) GetType() JobType
- func (*Job) ProtoMessage()
- func (x *Job) ProtoReflect() protoreflect.Message
- func (x *Job) Reset()
- func (x *Job) String() string
- type JobEventData
- type JobExecutionInfo
- func (*JobExecutionInfo) Descriptor() ([]byte, []int)deprecated
- func (x *JobExecutionInfo) GetStages() map[string]*JobExecutionStageInfo
- func (*JobExecutionInfo) ProtoMessage()
- func (x *JobExecutionInfo) ProtoReflect() protoreflect.Message
- func (x *JobExecutionInfo) Reset()
- func (x *JobExecutionInfo) String() string
- type JobExecutionStageInfo
- func (*JobExecutionStageInfo) Descriptor() ([]byte, []int)deprecated
- func (x *JobExecutionStageInfo) GetStepName() []string
- func (*JobExecutionStageInfo) ProtoMessage()
- func (x *JobExecutionStageInfo) ProtoReflect() protoreflect.Message
- func (x *JobExecutionStageInfo) Reset()
- func (x *JobExecutionStageInfo) String() string
- type JobMetadata
- func (*JobMetadata) Descriptor() ([]byte, []int)deprecated
- func (x *JobMetadata) GetBigTableDetails() []*BigTableIODetails
- func (x *JobMetadata) GetBigqueryDetails() []*BigQueryIODetails
- func (x *JobMetadata) GetDatastoreDetails() []*DatastoreIODetails
- func (x *JobMetadata) GetFileDetails() []*FileIODetails
- func (x *JobMetadata) GetPubsubDetails() []*PubSubIODetails
- func (x *JobMetadata) GetSdkVersion() *SdkVersion
- func (x *JobMetadata) GetSpannerDetails() []*SpannerIODetails
- func (*JobMetadata) ProtoMessage()
- func (x *JobMetadata) ProtoReflect() protoreflect.Message
- func (x *JobMetadata) Reset()
- func (x *JobMetadata) String() string
- type JobState
- type JobType
- type Package
- type PubSubIODetails
- func (*PubSubIODetails) Descriptor() ([]byte, []int)deprecated
- func (x *PubSubIODetails) GetSubscription() string
- func (x *PubSubIODetails) GetTopic() string
- func (*PubSubIODetails) ProtoMessage()
- func (x *PubSubIODetails) ProtoReflect() protoreflect.Message
- func (x *PubSubIODetails) Reset()
- func (x *PubSubIODetails) String() string
- type SdkHarnessContainerImage
- func (*SdkHarnessContainerImage) Descriptor() ([]byte, []int)deprecated
- func (x *SdkHarnessContainerImage) GetCapabilities() []string
- func (x *SdkHarnessContainerImage) GetContainerImage() string
- func (x *SdkHarnessContainerImage) GetEnvironmentId() string
- func (x *SdkHarnessContainerImage) GetUseSingleCorePerContainer() bool
- func (*SdkHarnessContainerImage) ProtoMessage()
- func (x *SdkHarnessContainerImage) ProtoReflect() protoreflect.Message
- func (x *SdkHarnessContainerImage) Reset()
- func (x *SdkHarnessContainerImage) String() string
- type SdkVersion
- func (*SdkVersion) Descriptor() ([]byte, []int)deprecated
- func (x *SdkVersion) GetSdkSupportStatus() SdkVersion_SdkSupportStatus
- func (x *SdkVersion) GetVersion() string
- func (x *SdkVersion) GetVersionDisplayName() string
- func (*SdkVersion) ProtoMessage()
- func (x *SdkVersion) ProtoReflect() protoreflect.Message
- func (x *SdkVersion) Reset()
- func (x *SdkVersion) String() string
- type SdkVersion_SdkSupportStatus
- func (SdkVersion_SdkSupportStatus) Descriptor() protoreflect.EnumDescriptor
- func (x SdkVersion_SdkSupportStatus) Enum() *SdkVersion_SdkSupportStatus
- func (SdkVersion_SdkSupportStatus) EnumDescriptor() ([]byte, []int)deprecated
- func (x SdkVersion_SdkSupportStatus) Number() protoreflect.EnumNumber
- func (x SdkVersion_SdkSupportStatus) String() string
- func (SdkVersion_SdkSupportStatus) Type() protoreflect.EnumType
- type ShuffleMode
- func (ShuffleMode) Descriptor() protoreflect.EnumDescriptor
- func (x ShuffleMode) Enum() *ShuffleMode
- func (ShuffleMode) EnumDescriptor() ([]byte, []int)deprecated
- func (x ShuffleMode) Number() protoreflect.EnumNumber
- func (x ShuffleMode) String() string
- func (ShuffleMode) Type() protoreflect.EnumType
- type SpannerIODetails
- func (*SpannerIODetails) Descriptor() ([]byte, []int)deprecated
- func (x *SpannerIODetails) GetDatabaseId() string
- func (x *SpannerIODetails) GetInstanceId() string
- func (x *SpannerIODetails) GetProjectId() string
- func (*SpannerIODetails) ProtoMessage()
- func (x *SpannerIODetails) ProtoReflect() protoreflect.Message
- func (x *SpannerIODetails) Reset()
- func (x *SpannerIODetails) String() string
- type TeardownPolicy
- func (TeardownPolicy) Descriptor() protoreflect.EnumDescriptor
- func (x TeardownPolicy) Enum() *TeardownPolicy
- func (TeardownPolicy) EnumDescriptor() ([]byte, []int)deprecated
- func (x TeardownPolicy) Number() protoreflect.EnumNumber
- func (x TeardownPolicy) String() string
- func (TeardownPolicy) Type() protoreflect.EnumType
- type WorkerIPAddressConfiguration
- func (WorkerIPAddressConfiguration) Descriptor() protoreflect.EnumDescriptor
- func (x WorkerIPAddressConfiguration) Enum() *WorkerIPAddressConfiguration
- func (WorkerIPAddressConfiguration) EnumDescriptor() ([]byte, []int)deprecated
- func (x WorkerIPAddressConfiguration) Number() protoreflect.EnumNumber
- func (x WorkerIPAddressConfiguration) String() string
- func (WorkerIPAddressConfiguration) Type() protoreflect.EnumType
- type WorkerPool
- func (*WorkerPool) Descriptor() ([]byte, []int)deprecated
- func (x *WorkerPool) GetAutoscalingSettings() *AutoscalingSettings
- func (x *WorkerPool) GetDefaultPackageSet() DefaultPackageSet
- func (x *WorkerPool) GetDiskSizeGb() int32
- func (x *WorkerPool) GetDiskSourceImage() string
- func (x *WorkerPool) GetDiskType() string
- func (x *WorkerPool) GetIpConfiguration() WorkerIPAddressConfiguration
- func (x *WorkerPool) GetKind() string
- func (x *WorkerPool) GetMachineType() string
- func (x *WorkerPool) GetMetadata() map[string]string
- func (x *WorkerPool) GetNetwork() string
- func (x *WorkerPool) GetNumThreadsPerWorker() int32
- func (x *WorkerPool) GetNumWorkers() int32
- func (x *WorkerPool) GetOnHostMaintenance() string
- func (x *WorkerPool) GetPackages() []*Package
- func (x *WorkerPool) GetSdkHarnessContainerImages() []*SdkHarnessContainerImage
- func (x *WorkerPool) GetSubnetwork() string
- func (x *WorkerPool) GetTeardownPolicy() TeardownPolicy
- func (x *WorkerPool) GetWorkerHarnessContainerImage() string
- func (x *WorkerPool) GetZone() string
- func (*WorkerPool) ProtoMessage()
- func (x *WorkerPool) ProtoReflect() protoreflect.Message
- func (x *WorkerPool) Reset()
- func (x *WorkerPool) String() string
Constants ¶
This section is empty.
Variables ¶
var ( JobType_name = map[int32]string{ 0: "JOB_TYPE_UNKNOWN", 1: "JOB_TYPE_BATCH", 2: "JOB_TYPE_STREAMING", } JobType_value = map[string]int32{ "JOB_TYPE_UNKNOWN": 0, "JOB_TYPE_BATCH": 1, "JOB_TYPE_STREAMING": 2, } )
Enum value maps for JobType.
var ( FlexResourceSchedulingGoal_name = map[int32]string{ 0: "FLEXRS_UNSPECIFIED", 1: "FLEXRS_SPEED_OPTIMIZED", 2: "FLEXRS_COST_OPTIMIZED", } FlexResourceSchedulingGoal_value = map[string]int32{ "FLEXRS_UNSPECIFIED": 0, "FLEXRS_SPEED_OPTIMIZED": 1, "FLEXRS_COST_OPTIMIZED": 2, } )
Enum value maps for FlexResourceSchedulingGoal.
var ( TeardownPolicy_name = map[int32]string{ 0: "TEARDOWN_POLICY_UNKNOWN", 1: "TEARDOWN_ALWAYS", 2: "TEARDOWN_ON_SUCCESS", 3: "TEARDOWN_NEVER", } TeardownPolicy_value = map[string]int32{ "TEARDOWN_POLICY_UNKNOWN": 0, "TEARDOWN_ALWAYS": 1, "TEARDOWN_ON_SUCCESS": 2, "TEARDOWN_NEVER": 3, } )
Enum value maps for TeardownPolicy.
var ( DefaultPackageSet_name = map[int32]string{ 0: "DEFAULT_PACKAGE_SET_UNKNOWN", 1: "DEFAULT_PACKAGE_SET_NONE", 2: "DEFAULT_PACKAGE_SET_JAVA", 3: "DEFAULT_PACKAGE_SET_PYTHON", } DefaultPackageSet_value = map[string]int32{ "DEFAULT_PACKAGE_SET_UNKNOWN": 0, "DEFAULT_PACKAGE_SET_NONE": 1, "DEFAULT_PACKAGE_SET_JAVA": 2, "DEFAULT_PACKAGE_SET_PYTHON": 3, } )
Enum value maps for DefaultPackageSet.
var ( AutoscalingAlgorithm_name = map[int32]string{ 0: "AUTOSCALING_ALGORITHM_UNKNOWN", 1: "AUTOSCALING_ALGORITHM_NONE", 2: "AUTOSCALING_ALGORITHM_BASIC", } AutoscalingAlgorithm_value = map[string]int32{ "AUTOSCALING_ALGORITHM_UNKNOWN": 0, "AUTOSCALING_ALGORITHM_NONE": 1, "AUTOSCALING_ALGORITHM_BASIC": 2, } )
Enum value maps for AutoscalingAlgorithm.
var ( WorkerIPAddressConfiguration_name = map[int32]string{ 0: "WORKER_IP_UNSPECIFIED", 1: "WORKER_IP_PUBLIC", 2: "WORKER_IP_PRIVATE", } WorkerIPAddressConfiguration_value = map[string]int32{ "WORKER_IP_UNSPECIFIED": 0, "WORKER_IP_PUBLIC": 1, "WORKER_IP_PRIVATE": 2, } )
Enum value maps for WorkerIPAddressConfiguration.
var ( ShuffleMode_name = map[int32]string{ 0: "SHUFFLE_MODE_UNSPECIFIED", 1: "VM_BASED", 2: "SERVICE_BASED", } ShuffleMode_value = map[string]int32{ "SHUFFLE_MODE_UNSPECIFIED": 0, "VM_BASED": 1, "SERVICE_BASED": 2, } )
Enum value maps for ShuffleMode.
var ( JobState_name = map[int32]string{ 0: "JOB_STATE_UNKNOWN", 1: "JOB_STATE_STOPPED", 2: "JOB_STATE_RUNNING", 3: "JOB_STATE_DONE", 4: "JOB_STATE_FAILED", 5: "JOB_STATE_CANCELLED", 6: "JOB_STATE_UPDATED", 7: "JOB_STATE_DRAINING", 8: "JOB_STATE_DRAINED", 9: "JOB_STATE_PENDING", 10: "JOB_STATE_CANCELLING", 11: "JOB_STATE_QUEUED", 12: "JOB_STATE_RESOURCE_CLEANING_UP", } JobState_value = map[string]int32{ "JOB_STATE_UNKNOWN": 0, "JOB_STATE_STOPPED": 1, "JOB_STATE_RUNNING": 2, "JOB_STATE_DONE": 3, "JOB_STATE_FAILED": 4, "JOB_STATE_CANCELLED": 5, "JOB_STATE_UPDATED": 6, "JOB_STATE_DRAINING": 7, "JOB_STATE_DRAINED": 8, "JOB_STATE_PENDING": 9, "JOB_STATE_CANCELLING": 10, "JOB_STATE_QUEUED": 11, "JOB_STATE_RESOURCE_CLEANING_UP": 12, } )
Enum value maps for JobState.
var ( SdkVersion_SdkSupportStatus_name = map[int32]string{ 0: "UNKNOWN", 1: "SUPPORTED", 2: "STALE", 3: "DEPRECATED", 4: "UNSUPPORTED", } SdkVersion_SdkSupportStatus_value = map[string]int32{ "UNKNOWN": 0, "SUPPORTED": 1, "STALE": 2, "DEPRECATED": 3, "UNSUPPORTED": 4, } )
Enum value maps for SdkVersion_SdkSupportStatus.
var File_cloud_dataflow_v1beta3_data_proto protoreflect.FileDescriptor
Functions ¶
This section is empty.
Types ¶
type AutoscalingAlgorithm ¶
type AutoscalingAlgorithm int32
Specifies the algorithm used to determine the number of worker processes to run at any given point in time, based on the amount of data left to process, the number of workers, and how quickly existing workers are processing data.
const ( // The algorithm is unknown, or unspecified. AutoscalingAlgorithm_AUTOSCALING_ALGORITHM_UNKNOWN AutoscalingAlgorithm = 0 // Disable autoscaling. AutoscalingAlgorithm_AUTOSCALING_ALGORITHM_NONE AutoscalingAlgorithm = 1 // Increase worker count over time to reduce job execution time. AutoscalingAlgorithm_AUTOSCALING_ALGORITHM_BASIC AutoscalingAlgorithm = 2 )
func (AutoscalingAlgorithm) Descriptor ¶
func (AutoscalingAlgorithm) Descriptor() protoreflect.EnumDescriptor
func (AutoscalingAlgorithm) Enum ¶
func (x AutoscalingAlgorithm) Enum() *AutoscalingAlgorithm
func (AutoscalingAlgorithm) EnumDescriptor
deprecated
func (AutoscalingAlgorithm) EnumDescriptor() ([]byte, []int)
Deprecated: Use AutoscalingAlgorithm.Descriptor instead.
func (AutoscalingAlgorithm) Number ¶
func (x AutoscalingAlgorithm) Number() protoreflect.EnumNumber
func (AutoscalingAlgorithm) String ¶
func (x AutoscalingAlgorithm) String() string
func (AutoscalingAlgorithm) Type ¶
func (AutoscalingAlgorithm) Type() protoreflect.EnumType
type AutoscalingSettings ¶
type AutoscalingSettings struct { // The algorithm to use for autoscaling. Algorithm AutoscalingAlgorithm `` /* 135-byte string literal not displayed */ // The maximum number of workers to cap scaling at. MaxNumWorkers int32 `protobuf:"varint,2,opt,name=max_num_workers,json=maxNumWorkers,proto3" json:"max_num_workers,omitempty"` // contains filtered or unexported fields }
Settings for WorkerPool autoscaling.
func (*AutoscalingSettings) Descriptor
deprecated
func (*AutoscalingSettings) Descriptor() ([]byte, []int)
Deprecated: Use AutoscalingSettings.ProtoReflect.Descriptor instead.
func (*AutoscalingSettings) GetAlgorithm ¶
func (x *AutoscalingSettings) GetAlgorithm() AutoscalingAlgorithm
func (*AutoscalingSettings) GetMaxNumWorkers ¶
func (x *AutoscalingSettings) GetMaxNumWorkers() int32
func (*AutoscalingSettings) ProtoMessage ¶
func (*AutoscalingSettings) ProtoMessage()
func (*AutoscalingSettings) ProtoReflect ¶
func (x *AutoscalingSettings) ProtoReflect() protoreflect.Message
func (*AutoscalingSettings) Reset ¶
func (x *AutoscalingSettings) Reset()
func (*AutoscalingSettings) String ¶
func (x *AutoscalingSettings) String() string
type BigQueryIODetails ¶
type BigQueryIODetails struct { // Table accessed in the connection. Table string `protobuf:"bytes,1,opt,name=table,proto3" json:"table,omitempty"` // Dataset accessed in the connection. Dataset string `protobuf:"bytes,2,opt,name=dataset,proto3" json:"dataset,omitempty"` // Project accessed in the connection. ProjectId string `protobuf:"bytes,3,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` // Query used to access data in the connection. Query string `protobuf:"bytes,4,opt,name=query,proto3" json:"query,omitempty"` // contains filtered or unexported fields }
Metadata for a BigQuery connector used by the job.
func (*BigQueryIODetails) Descriptor
deprecated
func (*BigQueryIODetails) Descriptor() ([]byte, []int)
Deprecated: Use BigQueryIODetails.ProtoReflect.Descriptor instead.
func (*BigQueryIODetails) GetDataset ¶
func (x *BigQueryIODetails) GetDataset() string
func (*BigQueryIODetails) GetProjectId ¶
func (x *BigQueryIODetails) GetProjectId() string
func (*BigQueryIODetails) GetQuery ¶
func (x *BigQueryIODetails) GetQuery() string
func (*BigQueryIODetails) GetTable ¶
func (x *BigQueryIODetails) GetTable() string
func (*BigQueryIODetails) ProtoMessage ¶
func (*BigQueryIODetails) ProtoMessage()
func (*BigQueryIODetails) ProtoReflect ¶
func (x *BigQueryIODetails) ProtoReflect() protoreflect.Message
func (*BigQueryIODetails) Reset ¶
func (x *BigQueryIODetails) Reset()
func (*BigQueryIODetails) String ¶
func (x *BigQueryIODetails) String() string
type BigTableIODetails ¶
type BigTableIODetails struct { // ProjectId accessed in the connection. ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` // InstanceId accessed in the connection. InstanceId string `protobuf:"bytes,2,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` // TableId accessed in the connection. TableId string `protobuf:"bytes,3,opt,name=table_id,json=tableId,proto3" json:"table_id,omitempty"` // contains filtered or unexported fields }
Metadata for a Cloud Bigtable connector used by the job.
func (*BigTableIODetails) Descriptor
deprecated
func (*BigTableIODetails) Descriptor() ([]byte, []int)
Deprecated: Use BigTableIODetails.ProtoReflect.Descriptor instead.
func (*BigTableIODetails) GetInstanceId ¶
func (x *BigTableIODetails) GetInstanceId() string
func (*BigTableIODetails) GetProjectId ¶
func (x *BigTableIODetails) GetProjectId() string
func (*BigTableIODetails) GetTableId ¶
func (x *BigTableIODetails) GetTableId() string
func (*BigTableIODetails) ProtoMessage ¶
func (*BigTableIODetails) ProtoMessage()
func (*BigTableIODetails) ProtoReflect ¶
func (x *BigTableIODetails) ProtoReflect() protoreflect.Message
func (*BigTableIODetails) Reset ¶
func (x *BigTableIODetails) Reset()
func (*BigTableIODetails) String ¶
func (x *BigTableIODetails) String() string
type DatastoreIODetails ¶
type DatastoreIODetails struct { // Namespace used in the connection. Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` // ProjectId accessed in the connection. ProjectId string `protobuf:"bytes,2,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` // contains filtered or unexported fields }
Metadata for a Datastore connector used by the job.
func (*DatastoreIODetails) Descriptor
deprecated
func (*DatastoreIODetails) Descriptor() ([]byte, []int)
Deprecated: Use DatastoreIODetails.ProtoReflect.Descriptor instead.
func (*DatastoreIODetails) GetNamespace ¶
func (x *DatastoreIODetails) GetNamespace() string
func (*DatastoreIODetails) GetProjectId ¶
func (x *DatastoreIODetails) GetProjectId() string
func (*DatastoreIODetails) ProtoMessage ¶
func (*DatastoreIODetails) ProtoMessage()
func (*DatastoreIODetails) ProtoReflect ¶
func (x *DatastoreIODetails) ProtoReflect() protoreflect.Message
func (*DatastoreIODetails) Reset ¶
func (x *DatastoreIODetails) Reset()
func (*DatastoreIODetails) String ¶
func (x *DatastoreIODetails) String() string
type DebugOptions ¶
type DebugOptions struct { // When true, enables the logging of the literal hot key to the user's Cloud // Logging. EnableHotKeyLogging bool `protobuf:"varint,1,opt,name=enable_hot_key_logging,json=enableHotKeyLogging,proto3" json:"enable_hot_key_logging,omitempty"` // contains filtered or unexported fields }
Describes any options that have an effect on the debugging of pipelines.
func (*DebugOptions) Descriptor
deprecated
func (*DebugOptions) Descriptor() ([]byte, []int)
Deprecated: Use DebugOptions.ProtoReflect.Descriptor instead.
func (*DebugOptions) GetEnableHotKeyLogging ¶
func (x *DebugOptions) GetEnableHotKeyLogging() bool
func (*DebugOptions) ProtoMessage ¶
func (*DebugOptions) ProtoMessage()
func (*DebugOptions) ProtoReflect ¶
func (x *DebugOptions) ProtoReflect() protoreflect.Message
func (*DebugOptions) Reset ¶
func (x *DebugOptions) Reset()
func (*DebugOptions) String ¶
func (x *DebugOptions) String() string
type DefaultPackageSet ¶
type DefaultPackageSet int32
The default set of packages to be staged on a pool of workers.
const ( // The default set of packages to stage is unknown, or unspecified. DefaultPackageSet_DEFAULT_PACKAGE_SET_UNKNOWN DefaultPackageSet = 0 // Indicates that no packages should be staged at the worker unless // explicitly specified by the job. DefaultPackageSet_DEFAULT_PACKAGE_SET_NONE DefaultPackageSet = 1 // Stage packages typically useful to workers written in Java. DefaultPackageSet_DEFAULT_PACKAGE_SET_JAVA DefaultPackageSet = 2 // Stage packages typically useful to workers written in Python. DefaultPackageSet_DEFAULT_PACKAGE_SET_PYTHON DefaultPackageSet = 3 )
func (DefaultPackageSet) Descriptor ¶
func (DefaultPackageSet) Descriptor() protoreflect.EnumDescriptor
func (DefaultPackageSet) Enum ¶
func (x DefaultPackageSet) Enum() *DefaultPackageSet
func (DefaultPackageSet) EnumDescriptor
deprecated
func (DefaultPackageSet) EnumDescriptor() ([]byte, []int)
Deprecated: Use DefaultPackageSet.Descriptor instead.
func (DefaultPackageSet) Number ¶
func (x DefaultPackageSet) Number() protoreflect.EnumNumber
func (DefaultPackageSet) String ¶
func (x DefaultPackageSet) String() string
func (DefaultPackageSet) Type ¶
func (DefaultPackageSet) Type() protoreflect.EnumType
type Environment ¶
type Environment struct { // The prefix of the resources the system should use for temporary // storage. The system will append the suffix "/temp-{JOBNAME} to // this resource prefix, where {JOBNAME} is the value of the // job_name field. The resulting bucket and object prefix is used // as the prefix of the resources used to store temporary data // needed during the job execution. NOTE: This will override the // value in taskrunner_settings. // The supported resource type is: // // Google Cloud Storage: // // storage.googleapis.com/{bucket}/{object} // bucket.storage.googleapis.com/{object} TempStoragePrefix string `protobuf:"bytes,1,opt,name=temp_storage_prefix,json=tempStoragePrefix,proto3" json:"temp_storage_prefix,omitempty"` // The type of cluster manager API to use. If unknown or // unspecified, the service will attempt to choose a reasonable // default. This should be in the form of the API service name, // e.g. "compute.googleapis.com". ClusterManagerApiService string `` /* 137-byte string literal not displayed */ // The list of experiments to enable. This field should be used for SDK // related experiments and not for service related experiments. The proper // field for service related experiments is service_options. Experiments []string `protobuf:"bytes,3,rep,name=experiments,proto3" json:"experiments,omitempty"` // The list of service options to enable. This field should be used for // service related experiments only. These experiments, when graduating to GA, // should be replaced by dedicated fields or become default (i.e. always on). ServiceOptions []string `protobuf:"bytes,16,rep,name=service_options,json=serviceOptions,proto3" json:"service_options,omitempty"` // If set, contains the Cloud KMS key identifier used to encrypt data // at rest, AKA a Customer Managed Encryption Key (CMEK). // // Format: // // projects/PROJECT_ID/locations/LOCATION/keyRings/KEY_RING/cryptoKeys/KEY ServiceKmsKeyName string `protobuf:"bytes,12,opt,name=service_kms_key_name,json=serviceKmsKeyName,proto3" json:"service_kms_key_name,omitempty"` // The worker pools. At least one "harness" worker pool must be // specified in order for the job to have workers. WorkerPools []*WorkerPool `protobuf:"bytes,4,rep,name=worker_pools,json=workerPools,proto3" json:"worker_pools,omitempty"` // A description of the process that generated the request. UserAgent *structpb.Struct `protobuf:"bytes,5,opt,name=user_agent,json=userAgent,proto3" json:"user_agent,omitempty"` // A structure describing which components and their versions of the service // are required in order to run the job. Version *structpb.Struct `protobuf:"bytes,6,opt,name=version,proto3" json:"version,omitempty"` // The dataset for the current project where various workflow // related tables are stored. // // The supported resource type is: // // Google BigQuery: // // bigquery.googleapis.com/{dataset} Dataset string `protobuf:"bytes,7,opt,name=dataset,proto3" json:"dataset,omitempty"` // The Cloud Dataflow SDK pipeline options specified by the user. These // options are passed through the service and are used to recreate the // SDK pipeline options on the worker in a language agnostic and platform // independent way. SdkPipelineOptions *structpb.Struct `protobuf:"bytes,8,opt,name=sdk_pipeline_options,json=sdkPipelineOptions,proto3" json:"sdk_pipeline_options,omitempty"` // Identity to run virtual machines as. Defaults to the default account. ServiceAccountEmail string `protobuf:"bytes,10,opt,name=service_account_email,json=serviceAccountEmail,proto3" json:"service_account_email,omitempty"` // Which Flexible Resource Scheduling mode to run in. FlexResourceSchedulingGoal FlexResourceSchedulingGoal `` /* 214-byte string literal not displayed */ // The Compute Engine region // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in // which worker processing should occur, e.g. "us-west1". Mutually exclusive // with worker_zone. If neither worker_region nor worker_zone is specified, // default to the control plane's region. WorkerRegion string `protobuf:"bytes,13,opt,name=worker_region,json=workerRegion,proto3" json:"worker_region,omitempty"` // The Compute Engine zone // (https://cloud.google.com/compute/docs/regions-zones/regions-zones) in // which worker processing should occur, e.g. "us-west1-a". Mutually exclusive // with worker_region. If neither worker_region nor worker_zone is specified, // a zone in the control plane's region is chosen based on available capacity. WorkerZone string `protobuf:"bytes,14,opt,name=worker_zone,json=workerZone,proto3" json:"worker_zone,omitempty"` // Output only. The shuffle mode used for the job. ShuffleMode ShuffleMode `` /* 150-byte string literal not displayed */ // Any debugging options to be supplied to the job. DebugOptions *DebugOptions `protobuf:"bytes,17,opt,name=debug_options,json=debugOptions,proto3" json:"debug_options,omitempty"` // contains filtered or unexported fields }
Describes the environment in which a Dataflow Job runs.
func (*Environment) Descriptor
deprecated
func (*Environment) Descriptor() ([]byte, []int)
Deprecated: Use Environment.ProtoReflect.Descriptor instead.
func (*Environment) GetClusterManagerApiService ¶
func (x *Environment) GetClusterManagerApiService() string
func (*Environment) GetDataset ¶
func (x *Environment) GetDataset() string
func (*Environment) GetDebugOptions ¶
func (x *Environment) GetDebugOptions() *DebugOptions
func (*Environment) GetExperiments ¶
func (x *Environment) GetExperiments() []string
func (*Environment) GetFlexResourceSchedulingGoal ¶
func (x *Environment) GetFlexResourceSchedulingGoal() FlexResourceSchedulingGoal
func (*Environment) GetSdkPipelineOptions ¶
func (x *Environment) GetSdkPipelineOptions() *structpb.Struct
func (*Environment) GetServiceAccountEmail ¶
func (x *Environment) GetServiceAccountEmail() string
func (*Environment) GetServiceKmsKeyName ¶
func (x *Environment) GetServiceKmsKeyName() string
func (*Environment) GetServiceOptions ¶
func (x *Environment) GetServiceOptions() []string
func (*Environment) GetShuffleMode ¶
func (x *Environment) GetShuffleMode() ShuffleMode
func (*Environment) GetTempStoragePrefix ¶
func (x *Environment) GetTempStoragePrefix() string
func (*Environment) GetUserAgent ¶
func (x *Environment) GetUserAgent() *structpb.Struct
func (*Environment) GetVersion ¶
func (x *Environment) GetVersion() *structpb.Struct
func (*Environment) GetWorkerPools ¶
func (x *Environment) GetWorkerPools() []*WorkerPool
func (*Environment) GetWorkerRegion ¶
func (x *Environment) GetWorkerRegion() string
func (*Environment) GetWorkerZone ¶
func (x *Environment) GetWorkerZone() string
func (*Environment) ProtoMessage ¶
func (*Environment) ProtoMessage()
func (*Environment) ProtoReflect ¶
func (x *Environment) ProtoReflect() protoreflect.Message
func (*Environment) Reset ¶
func (x *Environment) Reset()
func (*Environment) String ¶
func (x *Environment) String() string
type ExecutionStageState ¶
type ExecutionStageState struct { // The name of the execution stage. ExecutionStageName string `protobuf:"bytes,1,opt,name=execution_stage_name,json=executionStageName,proto3" json:"execution_stage_name,omitempty"` // Executions stage states allow the same set of values as JobState. ExecutionStageState JobState `` /* 172-byte string literal not displayed */ // The time at which the stage transitioned to this state. CurrentStateTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=current_state_time,json=currentStateTime,proto3" json:"current_state_time,omitempty"` // contains filtered or unexported fields }
A message describing the state of a particular execution stage.
func (*ExecutionStageState) Descriptor
deprecated
func (*ExecutionStageState) Descriptor() ([]byte, []int)
Deprecated: Use ExecutionStageState.ProtoReflect.Descriptor instead.
func (*ExecutionStageState) GetCurrentStateTime ¶
func (x *ExecutionStageState) GetCurrentStateTime() *timestamppb.Timestamp
func (*ExecutionStageState) GetExecutionStageName ¶
func (x *ExecutionStageState) GetExecutionStageName() string
func (*ExecutionStageState) GetExecutionStageState ¶
func (x *ExecutionStageState) GetExecutionStageState() JobState
func (*ExecutionStageState) ProtoMessage ¶
func (*ExecutionStageState) ProtoMessage()
func (*ExecutionStageState) ProtoReflect ¶
func (x *ExecutionStageState) ProtoReflect() protoreflect.Message
func (*ExecutionStageState) Reset ¶
func (x *ExecutionStageState) Reset()
func (*ExecutionStageState) String ¶
func (x *ExecutionStageState) String() string
type FileIODetails ¶
type FileIODetails struct { // File Pattern used to access files by the connector. FilePattern string `protobuf:"bytes,1,opt,name=file_pattern,json=filePattern,proto3" json:"file_pattern,omitempty"` // contains filtered or unexported fields }
Metadata for a File connector used by the job.
func (*FileIODetails) Descriptor
deprecated
func (*FileIODetails) Descriptor() ([]byte, []int)
Deprecated: Use FileIODetails.ProtoReflect.Descriptor instead.
func (*FileIODetails) GetFilePattern ¶
func (x *FileIODetails) GetFilePattern() string
func (*FileIODetails) ProtoMessage ¶
func (*FileIODetails) ProtoMessage()
func (*FileIODetails) ProtoReflect ¶
func (x *FileIODetails) ProtoReflect() protoreflect.Message
func (*FileIODetails) Reset ¶
func (x *FileIODetails) Reset()
func (*FileIODetails) String ¶
func (x *FileIODetails) String() string
type FlexResourceSchedulingGoal ¶
type FlexResourceSchedulingGoal int32
Specifies the resource to optimize for in Flexible Resource Scheduling.
const ( // Run in the default mode. FlexResourceSchedulingGoal_FLEXRS_UNSPECIFIED FlexResourceSchedulingGoal = 0 // Optimize for lower execution time. FlexResourceSchedulingGoal_FLEXRS_SPEED_OPTIMIZED FlexResourceSchedulingGoal = 1 // Optimize for lower cost. FlexResourceSchedulingGoal_FLEXRS_COST_OPTIMIZED FlexResourceSchedulingGoal = 2 )
func (FlexResourceSchedulingGoal) Descriptor ¶
func (FlexResourceSchedulingGoal) Descriptor() protoreflect.EnumDescriptor
func (FlexResourceSchedulingGoal) Enum ¶
func (x FlexResourceSchedulingGoal) Enum() *FlexResourceSchedulingGoal
func (FlexResourceSchedulingGoal) EnumDescriptor
deprecated
func (FlexResourceSchedulingGoal) EnumDescriptor() ([]byte, []int)
Deprecated: Use FlexResourceSchedulingGoal.Descriptor instead.
func (FlexResourceSchedulingGoal) Number ¶
func (x FlexResourceSchedulingGoal) Number() protoreflect.EnumNumber
func (FlexResourceSchedulingGoal) String ¶
func (x FlexResourceSchedulingGoal) String() string
func (FlexResourceSchedulingGoal) Type ¶
func (FlexResourceSchedulingGoal) Type() protoreflect.EnumType
type Job ¶
type Job struct { // The unique ID of this job. // // This field is set by the Cloud Dataflow service when the Job is // created, and is immutable for the life of the job. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` // The ID of the Cloud Platform project that the job belongs to. ProjectId string `protobuf:"bytes,2,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` // The user-specified Cloud Dataflow job name. // // Only one Job with a given name can exist in a project within one region at // any given time. Jobs in different regions can have the same name. // If a caller attempts to create a Job with the same // name as an already-existing Job, the attempt returns the // existing Job. // // The name must match the regular expression // `[a-z]([-a-z0-9]{0,1022}[a-z0-9])?` Name string `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` // The type of Cloud Dataflow job. Type JobType `protobuf:"varint,4,opt,name=type,proto3,enum=google.events.cloud.dataflow.v1beta3.JobType" json:"type,omitempty"` // The environment for the job. Environment *Environment `protobuf:"bytes,5,opt,name=environment,proto3" json:"environment,omitempty"` // The Cloud Storage location where the steps are stored. StepsLocation string `protobuf:"bytes,24,opt,name=steps_location,json=stepsLocation,proto3" json:"steps_location,omitempty"` // The current state of the job. // // Jobs are created in the `JOB_STATE_STOPPED` state unless otherwise // specified. // // A job in the `JOB_STATE_RUNNING` state may asynchronously enter a // terminal state. After a job has reached a terminal state, no // further state updates may be made. // // This field may be mutated by the Cloud Dataflow service; // callers cannot mutate it. CurrentState JobState `` /* 149-byte string literal not displayed */ // The timestamp associated with the current state. CurrentStateTime *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=current_state_time,json=currentStateTime,proto3" json:"current_state_time,omitempty"` // The job's requested state. // // `UpdateJob` may be used to switch between the `JOB_STATE_STOPPED` and // `JOB_STATE_RUNNING` states, by setting requested_state. `UpdateJob` may // also be used to directly set a job's requested state to // `JOB_STATE_CANCELLED` or `JOB_STATE_DONE`, irrevocably terminating the // job if it has not already reached a terminal state. RequestedState JobState `` /* 155-byte string literal not displayed */ // Deprecated. ExecutionInfo *JobExecutionInfo `protobuf:"bytes,10,opt,name=execution_info,json=executionInfo,proto3" json:"execution_info,omitempty"` // The timestamp when the job was initially created. Immutable and set by the // Cloud Dataflow service. CreateTime *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"` // If this job is an update of an existing job, this field is the job ID // of the job it replaced. // // When sending a `CreateJobRequest`, you can update a job by specifying it // here. The job named here is stopped, and its intermediate state is // transferred to this job. ReplaceJobId string `protobuf:"bytes,12,opt,name=replace_job_id,json=replaceJobId,proto3" json:"replace_job_id,omitempty"` // The client's unique identifier of the job, re-used across retried attempts. // If this field is set, the service will ensure its uniqueness. // The request to create a job will fail if the service has knowledge of a // previously submitted job with the same client's ID and job name. // The caller may use this field to ensure idempotence of job // creation across retried attempts to create a job. // By default, the field is empty and, in that case, the service ignores it. ClientRequestId string `protobuf:"bytes,14,opt,name=client_request_id,json=clientRequestId,proto3" json:"client_request_id,omitempty"` // If another job is an update of this job (and thus, this job is in // `JOB_STATE_UPDATED`), this field contains the ID of that job. ReplacedByJobId string `protobuf:"bytes,15,opt,name=replaced_by_job_id,json=replacedByJobId,proto3" json:"replaced_by_job_id,omitempty"` // A set of files the system should be aware of that are used // for temporary storage. These temporary files will be // removed on job completion. // No duplicates are allowed. // No file patterns are supported. // // The supported files are: // // Google Cloud Storage: // // storage.googleapis.com/{bucket}/{object} // bucket.storage.googleapis.com/{object} TempFiles []string `protobuf:"bytes,16,rep,name=temp_files,json=tempFiles,proto3" json:"temp_files,omitempty"` // User-defined labels for this job. // // The labels map can contain no more than 64 entries. Entries of the labels // map are UTF8 strings that comply with the following restrictions: // // * Keys must conform to regexp: [\p{Ll}\p{Lo}][\p{Ll}\p{Lo}\p{N}_-]{0,62} // * Values must conform to regexp: [\p{Ll}\p{Lo}\p{N}_-]{0,63} // * Both keys and values are additionally constrained to be <= 128 bytes in // size. Labels map[string]string `` /* 154-byte string literal not displayed */ // The [regional endpoint] // (https://cloud.google.com/dataflow/docs/concepts/regional-endpoints) that // contains this job. Location string `protobuf:"bytes,18,opt,name=location,proto3" json:"location,omitempty"` // This field may be mutated by the Cloud Dataflow service; // callers cannot mutate it. StageStates []*ExecutionStageState `protobuf:"bytes,20,rep,name=stage_states,json=stageStates,proto3" json:"stage_states,omitempty"` // This field is populated by the Dataflow service to support filtering jobs // by the metadata values provided here. Populated for ListJobs and all GetJob // views SUMMARY and higher. JobMetadata *JobMetadata `protobuf:"bytes,21,opt,name=job_metadata,json=jobMetadata,proto3" json:"job_metadata,omitempty"` // The timestamp when the job was started (transitioned to JOB_STATE_PENDING). // Flexible resource scheduling jobs are started with some delay after job // creation, so start_time is unset before start and is updated when the // job is started by the Cloud Dataflow service. For other jobs, start_time // always equals to create_time and is immutable and set by the Cloud Dataflow // service. StartTime *timestamppb.Timestamp `protobuf:"bytes,22,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` // If this is specified, the job's initial state is populated from the given // snapshot. CreatedFromSnapshotId string `` /* 129-byte string literal not displayed */ // Reserved for future use. This field is set only in responses from the // server; it is ignored if it is set in any requests. SatisfiesPzs bool `protobuf:"varint,25,opt,name=satisfies_pzs,json=satisfiesPzs,proto3" json:"satisfies_pzs,omitempty"` // contains filtered or unexported fields }
Defines a job to be run by the Cloud Dataflow service. Do not enter confidential information when you supply string values using the API. Fields stripped from source Job proto: - steps - pipeline_description - transform_name_mapping
func (*Job) Descriptor
deprecated
func (*Job) GetClientRequestId ¶
func (*Job) GetCreateTime ¶
func (x *Job) GetCreateTime() *timestamppb.Timestamp
func (*Job) GetCreatedFromSnapshotId ¶
func (*Job) GetCurrentState ¶
func (*Job) GetCurrentStateTime ¶
func (x *Job) GetCurrentStateTime() *timestamppb.Timestamp
func (*Job) GetEnvironment ¶
func (x *Job) GetEnvironment() *Environment
func (*Job) GetExecutionInfo ¶
func (x *Job) GetExecutionInfo() *JobExecutionInfo
func (*Job) GetJobMetadata ¶
func (x *Job) GetJobMetadata() *JobMetadata
func (*Job) GetLocation ¶
func (*Job) GetProjectId ¶
func (*Job) GetReplaceJobId ¶
func (*Job) GetReplacedByJobId ¶
func (*Job) GetRequestedState ¶
func (*Job) GetSatisfiesPzs ¶
func (*Job) GetStageStates ¶
func (x *Job) GetStageStates() []*ExecutionStageState
func (*Job) GetStartTime ¶
func (x *Job) GetStartTime() *timestamppb.Timestamp
func (*Job) GetStepsLocation ¶
func (*Job) GetTempFiles ¶
func (*Job) ProtoMessage ¶
func (*Job) ProtoMessage()
func (*Job) ProtoReflect ¶
func (x *Job) ProtoReflect() protoreflect.Message
type JobEventData ¶
type JobEventData struct { // The Job event payload. Payload *Job `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` // contains filtered or unexported fields }
The data within all Job events.
func (*JobEventData) Descriptor
deprecated
func (*JobEventData) Descriptor() ([]byte, []int)
Deprecated: Use JobEventData.ProtoReflect.Descriptor instead.
func (*JobEventData) GetPayload ¶
func (x *JobEventData) GetPayload() *Job
func (*JobEventData) ProtoMessage ¶
func (*JobEventData) ProtoMessage()
func (*JobEventData) ProtoReflect ¶
func (x *JobEventData) ProtoReflect() protoreflect.Message
func (*JobEventData) Reset ¶
func (x *JobEventData) Reset()
func (*JobEventData) String ¶
func (x *JobEventData) String() string
type JobExecutionInfo ¶
type JobExecutionInfo struct { // A mapping from each stage to the information about that stage. Stages map[string]*JobExecutionStageInfo `` /* 153-byte string literal not displayed */ // contains filtered or unexported fields }
Additional information about how a Cloud Dataflow job will be executed that isn't contained in the submitted job.
func (*JobExecutionInfo) Descriptor
deprecated
func (*JobExecutionInfo) Descriptor() ([]byte, []int)
Deprecated: Use JobExecutionInfo.ProtoReflect.Descriptor instead.
func (*JobExecutionInfo) GetStages ¶
func (x *JobExecutionInfo) GetStages() map[string]*JobExecutionStageInfo
func (*JobExecutionInfo) ProtoMessage ¶
func (*JobExecutionInfo) ProtoMessage()
func (*JobExecutionInfo) ProtoReflect ¶
func (x *JobExecutionInfo) ProtoReflect() protoreflect.Message
func (*JobExecutionInfo) Reset ¶
func (x *JobExecutionInfo) Reset()
func (*JobExecutionInfo) String ¶
func (x *JobExecutionInfo) String() string
type JobExecutionStageInfo ¶
type JobExecutionStageInfo struct { // The steps associated with the execution stage. // Note that stages may have several steps, and that a given step // might be run by more than one stage. StepName []string `protobuf:"bytes,1,rep,name=step_name,json=stepName,proto3" json:"step_name,omitempty"` // contains filtered or unexported fields }
Contains information about how a particular [google.dataflow.v1beta3.Step][google.dataflow.v1beta3.Step] will be executed.
func (*JobExecutionStageInfo) Descriptor
deprecated
func (*JobExecutionStageInfo) Descriptor() ([]byte, []int)
Deprecated: Use JobExecutionStageInfo.ProtoReflect.Descriptor instead.
func (*JobExecutionStageInfo) GetStepName ¶
func (x *JobExecutionStageInfo) GetStepName() []string
func (*JobExecutionStageInfo) ProtoMessage ¶
func (*JobExecutionStageInfo) ProtoMessage()
func (*JobExecutionStageInfo) ProtoReflect ¶
func (x *JobExecutionStageInfo) ProtoReflect() protoreflect.Message
func (*JobExecutionStageInfo) Reset ¶
func (x *JobExecutionStageInfo) Reset()
func (*JobExecutionStageInfo) String ¶
func (x *JobExecutionStageInfo) String() string
type JobMetadata ¶
type JobMetadata struct { // The SDK version used to run the job. SdkVersion *SdkVersion `protobuf:"bytes,1,opt,name=sdk_version,json=sdkVersion,proto3" json:"sdk_version,omitempty"` // Identification of a Spanner source used in the Dataflow job. SpannerDetails []*SpannerIODetails `protobuf:"bytes,2,rep,name=spanner_details,json=spannerDetails,proto3" json:"spanner_details,omitempty"` // Identification of a BigQuery source used in the Dataflow job. BigqueryDetails []*BigQueryIODetails `protobuf:"bytes,3,rep,name=bigquery_details,json=bigqueryDetails,proto3" json:"bigquery_details,omitempty"` // Identification of a Cloud Bigtable source used in the Dataflow job. BigTableDetails []*BigTableIODetails `protobuf:"bytes,4,rep,name=big_table_details,json=bigTableDetails,proto3" json:"big_table_details,omitempty"` // Identification of a Pub/Sub source used in the Dataflow job. PubsubDetails []*PubSubIODetails `protobuf:"bytes,5,rep,name=pubsub_details,json=pubsubDetails,proto3" json:"pubsub_details,omitempty"` // Identification of a File source used in the Dataflow job. FileDetails []*FileIODetails `protobuf:"bytes,6,rep,name=file_details,json=fileDetails,proto3" json:"file_details,omitempty"` // Identification of a Datastore source used in the Dataflow job. DatastoreDetails []*DatastoreIODetails `protobuf:"bytes,7,rep,name=datastore_details,json=datastoreDetails,proto3" json:"datastore_details,omitempty"` // contains filtered or unexported fields }
Metadata available primarily for filtering jobs. Will be included in the ListJob response and Job SUMMARY view.
func (*JobMetadata) Descriptor
deprecated
func (*JobMetadata) Descriptor() ([]byte, []int)
Deprecated: Use JobMetadata.ProtoReflect.Descriptor instead.
func (*JobMetadata) GetBigTableDetails ¶
func (x *JobMetadata) GetBigTableDetails() []*BigTableIODetails
func (*JobMetadata) GetBigqueryDetails ¶
func (x *JobMetadata) GetBigqueryDetails() []*BigQueryIODetails
func (*JobMetadata) GetDatastoreDetails ¶
func (x *JobMetadata) GetDatastoreDetails() []*DatastoreIODetails
func (*JobMetadata) GetFileDetails ¶
func (x *JobMetadata) GetFileDetails() []*FileIODetails
func (*JobMetadata) GetPubsubDetails ¶
func (x *JobMetadata) GetPubsubDetails() []*PubSubIODetails
func (*JobMetadata) GetSdkVersion ¶
func (x *JobMetadata) GetSdkVersion() *SdkVersion
func (*JobMetadata) GetSpannerDetails ¶
func (x *JobMetadata) GetSpannerDetails() []*SpannerIODetails
func (*JobMetadata) ProtoMessage ¶
func (*JobMetadata) ProtoMessage()
func (*JobMetadata) ProtoReflect ¶
func (x *JobMetadata) ProtoReflect() protoreflect.Message
func (*JobMetadata) Reset ¶
func (x *JobMetadata) Reset()
func (*JobMetadata) String ¶
func (x *JobMetadata) String() string
type JobState ¶
type JobState int32
Describes the overall state of a [google.dataflow.v1beta3.Job][google.dataflow.v1beta3.Job].
const ( // The job's run state isn't specified. JobState_JOB_STATE_UNKNOWN JobState = 0 // `JOB_STATE_STOPPED` indicates that the job has not // yet started to run. JobState_JOB_STATE_STOPPED JobState = 1 // `JOB_STATE_RUNNING` indicates that the job is currently running. JobState_JOB_STATE_RUNNING JobState = 2 // `JOB_STATE_DONE` indicates that the job has successfully completed. // This is a terminal job state. This state may be set by the Cloud Dataflow // service, as a transition from `JOB_STATE_RUNNING`. It may also be set via a // Cloud Dataflow `UpdateJob` call, if the job has not yet reached a terminal // state. JobState_JOB_STATE_DONE JobState = 3 // `JOB_STATE_FAILED` indicates that the job has failed. This is a // terminal job state. This state may only be set by the Cloud Dataflow // service, and only as a transition from `JOB_STATE_RUNNING`. JobState_JOB_STATE_FAILED JobState = 4 // `JOB_STATE_CANCELLED` indicates that the job has been explicitly // cancelled. This is a terminal job state. This state may only be // set via a Cloud Dataflow `UpdateJob` call, and only if the job has not // yet reached another terminal state. JobState_JOB_STATE_CANCELLED JobState = 5 // `JOB_STATE_UPDATED` indicates that the job was successfully updated, // meaning that this job was stopped and another job was started, inheriting // state from this one. This is a terminal job state. This state may only be // set by the Cloud Dataflow service, and only as a transition from // `JOB_STATE_RUNNING`. JobState_JOB_STATE_UPDATED JobState = 6 // `JOB_STATE_DRAINING` indicates that the job is in the process of draining. // A draining job has stopped pulling from its input sources and is processing // any data that remains in-flight. This state may be set via a Cloud Dataflow // `UpdateJob` call, but only as a transition from `JOB_STATE_RUNNING`. Jobs // that are draining may only transition to `JOB_STATE_DRAINED`, // `JOB_STATE_CANCELLED`, or `JOB_STATE_FAILED`. JobState_JOB_STATE_DRAINING JobState = 7 // `JOB_STATE_DRAINED` indicates that the job has been drained. // A drained job terminated by stopping pulling from its input sources and // processing any data that remained in-flight when draining was requested. // This state is a terminal state, may only be set by the Cloud Dataflow // service, and only as a transition from `JOB_STATE_DRAINING`. JobState_JOB_STATE_DRAINED JobState = 8 // `JOB_STATE_PENDING` indicates that the job has been created but is not yet // running. Jobs that are pending may only transition to `JOB_STATE_RUNNING`, // or `JOB_STATE_FAILED`. JobState_JOB_STATE_PENDING JobState = 9 // `JOB_STATE_CANCELLING` indicates that the job has been explicitly cancelled // and is in the process of stopping. Jobs that are cancelling may only // transition to `JOB_STATE_CANCELLED` or `JOB_STATE_FAILED`. JobState_JOB_STATE_CANCELLING JobState = 10 // `JOB_STATE_QUEUED` indicates that the job has been created but is being // delayed until launch. Jobs that are queued may only transition to // `JOB_STATE_PENDING` or `JOB_STATE_CANCELLED`. JobState_JOB_STATE_QUEUED JobState = 11 // `JOB_STATE_RESOURCE_CLEANING_UP` indicates that the batch job's associated // resources are currently being cleaned up after a successful run. // Currently, this is an opt-in feature, please reach out to Cloud support // team if you are interested. JobState_JOB_STATE_RESOURCE_CLEANING_UP JobState = 12 )
func (JobState) Descriptor ¶
func (JobState) Descriptor() protoreflect.EnumDescriptor
func (JobState) EnumDescriptor
deprecated
func (JobState) Number ¶
func (x JobState) Number() protoreflect.EnumNumber
func (JobState) Type ¶
func (JobState) Type() protoreflect.EnumType
type JobType ¶
type JobType int32
Specifies the processing model used by a [google.dataflow.v1beta3.Job], which determines the way the Job is managed by the Cloud Dataflow service (how workers are scheduled, how inputs are sharded, etc).
const ( // The type of the job is unspecified, or unknown. JobType_JOB_TYPE_UNKNOWN JobType = 0 // A batch job with a well-defined end point: data is read, data is // processed, data is written, and the job is done. JobType_JOB_TYPE_BATCH JobType = 1 // A continuously streaming job with no end: data is read, // processed, and written continuously. JobType_JOB_TYPE_STREAMING JobType = 2 )
func (JobType) Descriptor ¶
func (JobType) Descriptor() protoreflect.EnumDescriptor
func (JobType) EnumDescriptor
deprecated
func (JobType) Number ¶
func (x JobType) Number() protoreflect.EnumNumber
func (JobType) Type ¶
func (JobType) Type() protoreflect.EnumType
type Package ¶
type Package struct { // The name of the package. Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // The resource to read the package from. The supported resource type is: // // Google Cloud Storage: // // storage.googleapis.com/{bucket} // bucket.storage.googleapis.com/ Location string `protobuf:"bytes,2,opt,name=location,proto3" json:"location,omitempty"` // contains filtered or unexported fields }
The packages that must be installed in order for a worker to run the steps of the Cloud Dataflow job that will be assigned to its worker pool.
This is the mechanism by which the Cloud Dataflow SDK causes code to be loaded onto the workers. For example, the Cloud Dataflow Java SDK might use this to install jars containing the user's code and all of the various dependencies (libraries, data files, etc.) required in order for that code to run.
func (*Package) Descriptor
deprecated
func (*Package) GetLocation ¶
func (*Package) ProtoMessage ¶
func (*Package) ProtoMessage()
func (*Package) ProtoReflect ¶
func (x *Package) ProtoReflect() protoreflect.Message
type PubSubIODetails ¶
type PubSubIODetails struct { // Topic accessed in the connection. Topic string `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` // Subscription used in the connection. Subscription string `protobuf:"bytes,2,opt,name=subscription,proto3" json:"subscription,omitempty"` // contains filtered or unexported fields }
Metadata for a Pub/Sub connector used by the job.
func (*PubSubIODetails) Descriptor
deprecated
func (*PubSubIODetails) Descriptor() ([]byte, []int)
Deprecated: Use PubSubIODetails.ProtoReflect.Descriptor instead.
func (*PubSubIODetails) GetSubscription ¶
func (x *PubSubIODetails) GetSubscription() string
func (*PubSubIODetails) GetTopic ¶
func (x *PubSubIODetails) GetTopic() string
func (*PubSubIODetails) ProtoMessage ¶
func (*PubSubIODetails) ProtoMessage()
func (*PubSubIODetails) ProtoReflect ¶
func (x *PubSubIODetails) ProtoReflect() protoreflect.Message
func (*PubSubIODetails) Reset ¶
func (x *PubSubIODetails) Reset()
func (*PubSubIODetails) String ¶
func (x *PubSubIODetails) String() string
type SdkHarnessContainerImage ¶
type SdkHarnessContainerImage struct { // A docker container image that resides in Google Container Registry. ContainerImage string `protobuf:"bytes,1,opt,name=container_image,json=containerImage,proto3" json:"container_image,omitempty"` // If true, recommends the Dataflow service to use only one core per SDK // container instance with this image. If false (or unset) recommends using // more than one core per SDK container instance with this image for // efficiency. Note that Dataflow service may choose to override this property // if needed. UseSingleCorePerContainer bool `` /* 143-byte string literal not displayed */ // Environment ID for the Beam runner API proto Environment that corresponds // to the current SDK Harness. EnvironmentId string `protobuf:"bytes,3,opt,name=environment_id,json=environmentId,proto3" json:"environment_id,omitempty"` // The set of capabilities enumerated in the above Environment proto. See also // [beam_runner_api.proto](https://github.com/apache/beam/blob/master/model/pipeline/src/main/proto/org/apache/beam/model/pipeline/v1/beam_runner_api.proto) Capabilities []string `protobuf:"bytes,4,rep,name=capabilities,proto3" json:"capabilities,omitempty"` // contains filtered or unexported fields }
Defines an SDK harness container for executing Dataflow pipelines.
func (*SdkHarnessContainerImage) Descriptor
deprecated
func (*SdkHarnessContainerImage) Descriptor() ([]byte, []int)
Deprecated: Use SdkHarnessContainerImage.ProtoReflect.Descriptor instead.
func (*SdkHarnessContainerImage) GetCapabilities ¶
func (x *SdkHarnessContainerImage) GetCapabilities() []string
func (*SdkHarnessContainerImage) GetContainerImage ¶
func (x *SdkHarnessContainerImage) GetContainerImage() string
func (*SdkHarnessContainerImage) GetEnvironmentId ¶
func (x *SdkHarnessContainerImage) GetEnvironmentId() string
func (*SdkHarnessContainerImage) GetUseSingleCorePerContainer ¶
func (x *SdkHarnessContainerImage) GetUseSingleCorePerContainer() bool
func (*SdkHarnessContainerImage) ProtoMessage ¶
func (*SdkHarnessContainerImage) ProtoMessage()
func (*SdkHarnessContainerImage) ProtoReflect ¶
func (x *SdkHarnessContainerImage) ProtoReflect() protoreflect.Message
func (*SdkHarnessContainerImage) Reset ¶
func (x *SdkHarnessContainerImage) Reset()
func (*SdkHarnessContainerImage) String ¶
func (x *SdkHarnessContainerImage) String() string
type SdkVersion ¶
type SdkVersion struct { // The version of the SDK used to run the job. Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` // A readable string describing the version of the SDK. VersionDisplayName string `protobuf:"bytes,2,opt,name=version_display_name,json=versionDisplayName,proto3" json:"version_display_name,omitempty"` // The support status for this SDK version. SdkSupportStatus SdkVersion_SdkSupportStatus `` /* 182-byte string literal not displayed */ // contains filtered or unexported fields }
The version of the SDK used to run the job.
func (*SdkVersion) Descriptor
deprecated
func (*SdkVersion) Descriptor() ([]byte, []int)
Deprecated: Use SdkVersion.ProtoReflect.Descriptor instead.
func (*SdkVersion) GetSdkSupportStatus ¶
func (x *SdkVersion) GetSdkSupportStatus() SdkVersion_SdkSupportStatus
func (*SdkVersion) GetVersion ¶
func (x *SdkVersion) GetVersion() string
func (*SdkVersion) GetVersionDisplayName ¶
func (x *SdkVersion) GetVersionDisplayName() string
func (*SdkVersion) ProtoMessage ¶
func (*SdkVersion) ProtoMessage()
func (*SdkVersion) ProtoReflect ¶
func (x *SdkVersion) ProtoReflect() protoreflect.Message
func (*SdkVersion) Reset ¶
func (x *SdkVersion) Reset()
func (*SdkVersion) String ¶
func (x *SdkVersion) String() string
type SdkVersion_SdkSupportStatus ¶
type SdkVersion_SdkSupportStatus int32
The support status of the SDK used to run the job.
const ( // Cloud Dataflow is unaware of this version. SdkVersion_UNKNOWN SdkVersion_SdkSupportStatus = 0 // This is a known version of an SDK, and is supported. SdkVersion_SUPPORTED SdkVersion_SdkSupportStatus = 1 // A newer version of the SDK family exists, and an update is recommended. SdkVersion_STALE SdkVersion_SdkSupportStatus = 2 // This version of the SDK is deprecated and will eventually be // unsupported. SdkVersion_DEPRECATED SdkVersion_SdkSupportStatus = 3 // Support for this SDK version has ended and it should no longer be used. SdkVersion_UNSUPPORTED SdkVersion_SdkSupportStatus = 4 )
func (SdkVersion_SdkSupportStatus) Descriptor ¶
func (SdkVersion_SdkSupportStatus) Descriptor() protoreflect.EnumDescriptor
func (SdkVersion_SdkSupportStatus) Enum ¶
func (x SdkVersion_SdkSupportStatus) Enum() *SdkVersion_SdkSupportStatus
func (SdkVersion_SdkSupportStatus) EnumDescriptor
deprecated
func (SdkVersion_SdkSupportStatus) EnumDescriptor() ([]byte, []int)
Deprecated: Use SdkVersion_SdkSupportStatus.Descriptor instead.
func (SdkVersion_SdkSupportStatus) Number ¶
func (x SdkVersion_SdkSupportStatus) Number() protoreflect.EnumNumber
func (SdkVersion_SdkSupportStatus) String ¶
func (x SdkVersion_SdkSupportStatus) String() string
func (SdkVersion_SdkSupportStatus) Type ¶
func (SdkVersion_SdkSupportStatus) Type() protoreflect.EnumType
type ShuffleMode ¶
type ShuffleMode int32
Specifies the shuffle mode used by a [google.dataflow.v1beta3.Job], which determines the approach data is shuffled during processing. More details in: https://cloud.google.com/dataflow/docs/guides/deploying-a-pipeline#dataflow-shuffle
const ( // Shuffle mode information is not available. ShuffleMode_SHUFFLE_MODE_UNSPECIFIED ShuffleMode = 0 // Shuffle is done on the worker VMs. ShuffleMode_VM_BASED ShuffleMode = 1 // Shuffle is done on the service side. ShuffleMode_SERVICE_BASED ShuffleMode = 2 )
func (ShuffleMode) Descriptor ¶
func (ShuffleMode) Descriptor() protoreflect.EnumDescriptor
func (ShuffleMode) Enum ¶
func (x ShuffleMode) Enum() *ShuffleMode
func (ShuffleMode) EnumDescriptor
deprecated
func (ShuffleMode) EnumDescriptor() ([]byte, []int)
Deprecated: Use ShuffleMode.Descriptor instead.
func (ShuffleMode) Number ¶
func (x ShuffleMode) Number() protoreflect.EnumNumber
func (ShuffleMode) String ¶
func (x ShuffleMode) String() string
func (ShuffleMode) Type ¶
func (ShuffleMode) Type() protoreflect.EnumType
type SpannerIODetails ¶
type SpannerIODetails struct { // ProjectId accessed in the connection. ProjectId string `protobuf:"bytes,1,opt,name=project_id,json=projectId,proto3" json:"project_id,omitempty"` // InstanceId accessed in the connection. InstanceId string `protobuf:"bytes,2,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"` // DatabaseId accessed in the connection. DatabaseId string `protobuf:"bytes,3,opt,name=database_id,json=databaseId,proto3" json:"database_id,omitempty"` // contains filtered or unexported fields }
Metadata for a Spanner connector used by the job.
func (*SpannerIODetails) Descriptor
deprecated
func (*SpannerIODetails) Descriptor() ([]byte, []int)
Deprecated: Use SpannerIODetails.ProtoReflect.Descriptor instead.
func (*SpannerIODetails) GetDatabaseId ¶
func (x *SpannerIODetails) GetDatabaseId() string
func (*SpannerIODetails) GetInstanceId ¶
func (x *SpannerIODetails) GetInstanceId() string
func (*SpannerIODetails) GetProjectId ¶
func (x *SpannerIODetails) GetProjectId() string
func (*SpannerIODetails) ProtoMessage ¶
func (*SpannerIODetails) ProtoMessage()
func (*SpannerIODetails) ProtoReflect ¶
func (x *SpannerIODetails) ProtoReflect() protoreflect.Message
func (*SpannerIODetails) Reset ¶
func (x *SpannerIODetails) Reset()
func (*SpannerIODetails) String ¶
func (x *SpannerIODetails) String() string
type TeardownPolicy ¶
type TeardownPolicy int32
Specifies what happens to a resource when a Cloud Dataflow [google.dataflow.v1beta3.Job][google.dataflow.v1beta3.Job] has completed.
const ( // The teardown policy isn't specified, or is unknown. TeardownPolicy_TEARDOWN_POLICY_UNKNOWN TeardownPolicy = 0 // Always teardown the resource. TeardownPolicy_TEARDOWN_ALWAYS TeardownPolicy = 1 // Teardown the resource on success. This is useful for debugging // failures. TeardownPolicy_TEARDOWN_ON_SUCCESS TeardownPolicy = 2 // Never teardown the resource. This is useful for debugging and // development. TeardownPolicy_TEARDOWN_NEVER TeardownPolicy = 3 )
func (TeardownPolicy) Descriptor ¶
func (TeardownPolicy) Descriptor() protoreflect.EnumDescriptor
func (TeardownPolicy) Enum ¶
func (x TeardownPolicy) Enum() *TeardownPolicy
func (TeardownPolicy) EnumDescriptor
deprecated
func (TeardownPolicy) EnumDescriptor() ([]byte, []int)
Deprecated: Use TeardownPolicy.Descriptor instead.
func (TeardownPolicy) Number ¶
func (x TeardownPolicy) Number() protoreflect.EnumNumber
func (TeardownPolicy) String ¶
func (x TeardownPolicy) String() string
func (TeardownPolicy) Type ¶
func (TeardownPolicy) Type() protoreflect.EnumType
type WorkerIPAddressConfiguration ¶
type WorkerIPAddressConfiguration int32
Specifies how IP addresses should be allocated to the worker machines.
const ( // The configuration is unknown, or unspecified. WorkerIPAddressConfiguration_WORKER_IP_UNSPECIFIED WorkerIPAddressConfiguration = 0 // Workers should have public IP addresses. WorkerIPAddressConfiguration_WORKER_IP_PUBLIC WorkerIPAddressConfiguration = 1 // Workers should have private IP addresses. WorkerIPAddressConfiguration_WORKER_IP_PRIVATE WorkerIPAddressConfiguration = 2 )
func (WorkerIPAddressConfiguration) Descriptor ¶
func (WorkerIPAddressConfiguration) Descriptor() protoreflect.EnumDescriptor
func (WorkerIPAddressConfiguration) Enum ¶
func (x WorkerIPAddressConfiguration) Enum() *WorkerIPAddressConfiguration
func (WorkerIPAddressConfiguration) EnumDescriptor
deprecated
func (WorkerIPAddressConfiguration) EnumDescriptor() ([]byte, []int)
Deprecated: Use WorkerIPAddressConfiguration.Descriptor instead.
func (WorkerIPAddressConfiguration) Number ¶
func (x WorkerIPAddressConfiguration) Number() protoreflect.EnumNumber
func (WorkerIPAddressConfiguration) String ¶
func (x WorkerIPAddressConfiguration) String() string
func (WorkerIPAddressConfiguration) Type ¶
func (WorkerIPAddressConfiguration) Type() protoreflect.EnumType
type WorkerPool ¶
type WorkerPool struct { // The kind of the worker pool; currently only `harness` and `shuffle` // are supported. Kind string `protobuf:"bytes,1,opt,name=kind,proto3" json:"kind,omitempty"` // Number of Google Compute Engine workers in this pool needed to // execute the job. If zero or unspecified, the service will // attempt to choose a reasonable default. NumWorkers int32 `protobuf:"varint,2,opt,name=num_workers,json=numWorkers,proto3" json:"num_workers,omitempty"` // Packages to be installed on workers. Packages []*Package `protobuf:"bytes,3,rep,name=packages,proto3" json:"packages,omitempty"` // The default package set to install. This allows the service to // select a default set of packages which are useful to worker // harnesses written in a particular language. DefaultPackageSet DefaultPackageSet `` /* 175-byte string literal not displayed */ // Machine type (e.g. "n1-standard-1"). If empty or unspecified, the // service will attempt to choose a reasonable default. MachineType string `protobuf:"bytes,5,opt,name=machine_type,json=machineType,proto3" json:"machine_type,omitempty"` // Sets the policy for determining when to turndown worker pool. // Allowed values are: `TEARDOWN_ALWAYS`, `TEARDOWN_ON_SUCCESS`, and // `TEARDOWN_NEVER`. // `TEARDOWN_ALWAYS` means workers are always torn down regardless of whether // the job succeeds. `TEARDOWN_ON_SUCCESS` means workers are torn down // if the job succeeds. `TEARDOWN_NEVER` means the workers are never torn // down. // // If the workers are not torn down by the service, they will // continue to run and use Google Compute Engine VM resources in the // user's project until they are explicitly terminated by the user. // Because of this, Google recommends using the `TEARDOWN_ALWAYS` // policy except for small, manually supervised test jobs. // // If unknown or unspecified, the service will attempt to choose a reasonable // default. TeardownPolicy TeardownPolicy `` /* 161-byte string literal not displayed */ // Size of root disk for VMs, in GB. If zero or unspecified, the service will // attempt to choose a reasonable default. DiskSizeGb int32 `protobuf:"varint,7,opt,name=disk_size_gb,json=diskSizeGb,proto3" json:"disk_size_gb,omitempty"` // Type of root disk for VMs. If empty or unspecified, the service will // attempt to choose a reasonable default. DiskType string `protobuf:"bytes,16,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` // Fully qualified source image for disks. DiskSourceImage string `protobuf:"bytes,8,opt,name=disk_source_image,json=diskSourceImage,proto3" json:"disk_source_image,omitempty"` // Zone to run the worker pools in. If empty or unspecified, the service // will attempt to choose a reasonable default. Zone string `protobuf:"bytes,9,opt,name=zone,proto3" json:"zone,omitempty"` // The action to take on host maintenance, as defined by the Google // Compute Engine API. OnHostMaintenance string `protobuf:"bytes,11,opt,name=on_host_maintenance,json=onHostMaintenance,proto3" json:"on_host_maintenance,omitempty"` // Metadata to set on the Google Compute Engine VMs. Metadata map[string]string `` /* 158-byte string literal not displayed */ // Settings for autoscaling of this WorkerPool. AutoscalingSettings *AutoscalingSettings `protobuf:"bytes,14,opt,name=autoscaling_settings,json=autoscalingSettings,proto3" json:"autoscaling_settings,omitempty"` // Network to which VMs will be assigned. If empty or unspecified, // the service will use the network "default". Network string `protobuf:"bytes,17,opt,name=network,proto3" json:"network,omitempty"` // Subnetwork to which VMs will be assigned, if desired. Expected to be of // the form "regions/REGION/subnetworks/SUBNETWORK". Subnetwork string `protobuf:"bytes,19,opt,name=subnetwork,proto3" json:"subnetwork,omitempty"` // Required. Docker container image that executes the Cloud Dataflow worker // harness, residing in Google Container Registry. // // Deprecated for the Fn API path. Use sdk_harness_container_images instead. WorkerHarnessContainerImage string `` /* 147-byte string literal not displayed */ // The number of threads per worker harness. If empty or unspecified, the // service will choose a number of threads (according to the number of cores // on the selected machine type for batch, or 1 by convention for streaming). NumThreadsPerWorker int32 `protobuf:"varint,20,opt,name=num_threads_per_worker,json=numThreadsPerWorker,proto3" json:"num_threads_per_worker,omitempty"` // Configuration for VM IPs. IpConfiguration WorkerIPAddressConfiguration `` /* 179-byte string literal not displayed */ // Set of SDK harness containers needed to execute this pipeline. This will // only be set in the Fn API path. For non-cross-language pipelines this // should have only one entry. Cross-language pipelines will have two or more // entries. SdkHarnessContainerImages []*SdkHarnessContainerImage `` /* 141-byte string literal not displayed */ // contains filtered or unexported fields }
Describes one particular pool of Cloud Dataflow workers to be instantiated by the Cloud Dataflow service in order to perform the computations required by a job. Note that a workflow job may use multiple pools, in order to match the various computational requirements of the various stages of the job.
func (*WorkerPool) Descriptor
deprecated
func (*WorkerPool) Descriptor() ([]byte, []int)
Deprecated: Use WorkerPool.ProtoReflect.Descriptor instead.
func (*WorkerPool) GetAutoscalingSettings ¶
func (x *WorkerPool) GetAutoscalingSettings() *AutoscalingSettings
func (*WorkerPool) GetDefaultPackageSet ¶
func (x *WorkerPool) GetDefaultPackageSet() DefaultPackageSet
func (*WorkerPool) GetDiskSizeGb ¶
func (x *WorkerPool) GetDiskSizeGb() int32
func (*WorkerPool) GetDiskSourceImage ¶
func (x *WorkerPool) GetDiskSourceImage() string
func (*WorkerPool) GetDiskType ¶
func (x *WorkerPool) GetDiskType() string
func (*WorkerPool) GetIpConfiguration ¶
func (x *WorkerPool) GetIpConfiguration() WorkerIPAddressConfiguration
func (*WorkerPool) GetKind ¶
func (x *WorkerPool) GetKind() string
func (*WorkerPool) GetMachineType ¶
func (x *WorkerPool) GetMachineType() string
func (*WorkerPool) GetMetadata ¶
func (x *WorkerPool) GetMetadata() map[string]string
func (*WorkerPool) GetNetwork ¶
func (x *WorkerPool) GetNetwork() string
func (*WorkerPool) GetNumThreadsPerWorker ¶
func (x *WorkerPool) GetNumThreadsPerWorker() int32
func (*WorkerPool) GetNumWorkers ¶
func (x *WorkerPool) GetNumWorkers() int32
func (*WorkerPool) GetOnHostMaintenance ¶
func (x *WorkerPool) GetOnHostMaintenance() string
func (*WorkerPool) GetPackages ¶
func (x *WorkerPool) GetPackages() []*Package
func (*WorkerPool) GetSdkHarnessContainerImages ¶
func (x *WorkerPool) GetSdkHarnessContainerImages() []*SdkHarnessContainerImage
func (*WorkerPool) GetSubnetwork ¶
func (x *WorkerPool) GetSubnetwork() string
func (*WorkerPool) GetTeardownPolicy ¶
func (x *WorkerPool) GetTeardownPolicy() TeardownPolicy
func (*WorkerPool) GetWorkerHarnessContainerImage ¶
func (x *WorkerPool) GetWorkerHarnessContainerImage() string
func (*WorkerPool) GetZone ¶
func (x *WorkerPool) GetZone() string
func (*WorkerPool) ProtoMessage ¶
func (*WorkerPool) ProtoMessage()
func (*WorkerPool) ProtoReflect ¶
func (x *WorkerPool) ProtoReflect() protoreflect.Message
func (*WorkerPool) Reset ¶
func (x *WorkerPool) Reset()
func (*WorkerPool) String ¶
func (x *WorkerPool) String() string