client

Documentation

Index

• Variables
• func NewExecuteResponseStream(responseClient proto.SparkConnectService_ExecutePlanClient, sessionId string, ...) base.ExecuteResponseStream
• func NewRetriableSparkConnectClient(conn *grpc.ClientConn, sessionId string, opts options.SparkClientOptions) base.SparkConnectRPCClient
• func NewRuntimeConfig(client *base.SparkConnectClient) *runtimeConfig
• func NewSparkExecutor(conn *grpc.ClientConn, md metadata.MD, sessionId string, ...) base.SparkConnectClient
• func NewSparkExecutorFromClient(client base.SparkConnectRPCClient, md metadata.MD, sessionId string) base.SparkConnectClient
• func NewTestConnectClientFromResponses(sessionId string, r ...*mocks.MockResponse) base.SparkConnectClient
• func NewTestConnectClientWithImmediateError(sessionId string, err error) base.SparkConnectClient
• type ExecutePlanClient
  • func (c *ExecutePlanClient) Properties() map[string]any
  • func (c *ExecutePlanClient) ToTable() (*types.StructType, arrow.Table, error)
• type RetryHandler
• type RetryPolicy
• type RuntimeConfig

Variables

var DefaultRetryPolicy = RetryPolicy{
	MaxRetries:         15,
	InitialBackoff:     50 * time.Millisecond,
	MaxBackoff:         1 * time.Minute,
	BackoffMultiplier:  4.0,
	Jitter:             500 * time.Millisecond,
	MinJitterThreshold: 2000 * time.Millisecond,
	Name:               "DefaultRetryPolicy",
	Handler: func(e error) bool {
		status := sparkerrors.FromRPCError(e)
		switch status.Code {
		case codes.Unavailable:
			return true
		case codes.Internal:
			if strings.Contains(status.Message, "INVALID_CURSOR.DISCONNECTED") {
				return true
			}
		}
		return false
	},
}

DefaultRetryPolicy is the default retry policy used by the client. It will retry on Unavailable and in case the cursor has been disconnected. All other errors are considered to be not retriable.

var DefaultRetryPolicyRegistry = []RetryPolicy{DefaultRetryPolicy}

DefaultRetryPolicyRegistry is the default set of retry policies used by the client. It contains all those policies that are enabled by default.

var TestingRetryPolicy = RetryPolicy{
	MaxRetries:         5,
	InitialBackoff:     0,
	MaxBackoff:         1,
	BackoffMultiplier:  2,
	Jitter:             0,
	MinJitterThreshold: 0,
	Name:               "TestingRetryPolicy",
	Handler: func(e error) bool {
		status := sparkerrors.FromRPCError(e)
		switch status.Code {
		case codes.Unavailable:
			return true
		case codes.Internal:
			if strings.Contains(status.Message, "INVALID_CURSOR.DISCONNECTED") {
				return true
			}
		}
		return false
	},
}

Functions

func NewExecuteResponseStream

func NewExecuteResponseStream(
	responseClient proto.SparkConnectService_ExecutePlanClient,
	sessionId string,
	operationId string,
	opts options.SparkClientOptions,
) base.ExecuteResponseStream

func NewRetriableSparkConnectClient

func NewRetriableSparkConnectClient(conn *grpc.ClientConn, sessionId string,
	opts options.SparkClientOptions,
) base.SparkConnectRPCClient

func NewRuntimeConfig

func NewRuntimeConfig(client *base.SparkConnectClient) *runtimeConfig

Constructor for runtimeConfig used by SparkSession

func NewSparkExecutor

func NewSparkExecutor(conn *grpc.ClientConn, md metadata.MD, sessionId string, opts options.SparkClientOptions) base.SparkConnectClient

func NewSparkExecutorFromClient

func NewSparkExecutorFromClient(client base.SparkConnectRPCClient, md metadata.MD, sessionId string) base.SparkConnectClient

NewSparkExecutorFromClient creates a new SparkConnectClient from an existing client and is mostly used in testing.

func NewTestConnectClientFromResponses

func NewTestConnectClientFromResponses(sessionId string, r ...*mocks.MockResponse) base.SparkConnectClient

func NewTestConnectClientWithImmediateError

func NewTestConnectClientWithImmediateError(sessionId string, err error) base.SparkConnectClient

Types

type ExecutePlanClient

type ExecutePlanClient struct {
	// contains filtered or unexported fields
}

ExecutePlanClient is the wrapper around the result of the execution of a query plan using Spark Connect.

func (*ExecutePlanClient) Properties

func (c *ExecutePlanClient) Properties() map[string]any

func (*ExecutePlanClient) ToTable

func (c *ExecutePlanClient) ToTable() (*types.StructType, arrow.Table, error)

ToTable converts the result of the execution of a query plan to an Arrow Table.

type RetryHandler

type RetryHandler func(error) bool

type RetryPolicy

type RetryPolicy struct {
	MaxRetries         int32
	InitialBackoff     time.Duration
	MaxBackoff         time.Duration
	BackoffMultiplier  float32
	Jitter             time.Duration
	MinJitterThreshold time.Duration
	Name               string
	Handler            RetryHandler
}

RetryPolicy defines the parameters for a retry policy. The policy is used to determine if an error is retriable and how to handle retries. The policy defines the behavior of the client in how it backs off in case of an error and how the retries are spread out over time.

type RuntimeConfig

type RuntimeConfig interface {
	GetAll(ctx context.Context) (map[string]string, error)
	Set(ctx context.Context, key string, value string) error
	Get(ctx context.Context, key string) (string, error)
	Unset(ctx context.Context, key string) error
	IsModifiable(ctx context.Context, key string) (bool, error)
	GetWithDefault(ctx context.Context, key string, default_value string) (string, error)
}

Public interface RuntimeConfig