dataproc

Documentation

Index

• Constants
• type PySparkJob
  • func (j *PySparkJob) Command() *exec.Command
  • func (j *PySparkJob) Run(ctx context.Context) (string, error)
  • func (j *PySparkJob) Submit(ctx context.Context) (string, error)
  • func (j *PySparkJob) Wait(ctx context.Context) (string, error)

Constants

const (
	CLUSTER_SKIA  = "cluster-173d"
	JOB_ID_PREFIX = "  jobId: "
)

Types

type PySparkJob

type PySparkJob struct {
	// The main Python file to run.
	PyFile string
	// Any additional arguments to pass onto the job command line.
	Args []string
	// Which cluster should run the job.
	Cluster string
	// Any files to provide to the job.
	Files []string
	// contains filtered or unexported fields
}

PySparkJob describes a PySpark job which runs on a GCE cluster.

func (*PySparkJob) Command

func (j *PySparkJob) Command() *exec.Command

Return the command used to trigger this job.

func (*PySparkJob) Run

func (j *PySparkJob) Run(ctx context.Context) (string, error)

Run the job and return its output.

func (*PySparkJob) Submit

func (j *PySparkJob) Submit(ctx context.Context) (string, error)

Trigger the job and return its ID.

func (*PySparkJob) Wait

func (j *PySparkJob) Wait(ctx context.Context) (string, error)

Wait for the job to complete and return its output.