sql

Documentation

Index

• Variables
• type BinaryType
  • func (t BinaryType) TypeName() string
• type BooleanType
  • func (t BooleanType) TypeName() string
• type ByteType
  • func (t ByteType) TypeName() string
• type DataFrame
• type DataFrameReader
• type DataFrameWriter
• type DataType
• type DateType
  • func (t DateType) TypeName() string
• type DecimalType
  • func (t DecimalType) TypeName() string
• type DoubleType
  • func (t DoubleType) TypeName() string
• type FloatType
  • func (t FloatType) TypeName() string
• type GenericRowWithSchema
  • func (r *GenericRowWithSchema) Schema() (*StructType, error)
  • func (r *GenericRowWithSchema) Values() ([]any, error)
• type IntegerType
  • func (t IntegerType) TypeName() string
• type LongType
  • func (t LongType) TypeName() string
• type RangePartitionColumn
• type Row
• type ShortType
  • func (t ShortType) TypeName() string
• type SparkSessionBuilder
  • func (s SparkSessionBuilder) Build() (sparkSession, error)
  • func (s SparkSessionBuilder) Remote(connectionString string) SparkSessionBuilder
• type StringType
  • func (t StringType) TypeName() string
• type StructField
• type StructType
• type TimestampNtzType
  • func (t TimestampNtzType) TypeName() string
• type TimestampType
  • func (t TimestampType) TypeName() string
• type UnsupportedType
  • func (t UnsupportedType) TypeName() string

Variables

var SparkSession sparkSessionBuilderEntrypoint

Types

type BinaryType

type BinaryType struct {
}

func (BinaryType) TypeName

func (t BinaryType) TypeName() string

type BooleanType

type BooleanType struct {
}

func (BooleanType) TypeName

func (t BooleanType) TypeName() string

type ByteType

type ByteType struct {
}

func (ByteType) TypeName

func (t ByteType) TypeName() string

type DataFrame

type DataFrame interface {
	// Show prints out data frame data.
	Show(numRows int, truncate bool) error
	// Schema returns the schema for the current data frame.
	Schema() (*StructType, error)
	// Collect returns the data rows of the current data frame.
	Collect() ([]Row, error)
	// Write returns a data frame writer, which could be used to save data frame to supported storage.
	Write() DataFrameWriter
	// CreateTempView creates or replaces a temporary view.
	CreateTempView(viewName string, replace bool, global bool) error
	// Repartition re-partitions a data frame.
	Repartition(numPartitions int, columns []string) (DataFrame, error)
	// RepartitionByRange re-partitions a data frame by range partition.
	RepartitionByRange(numPartitions int, columns []RangePartitionColumn) (DataFrame, error)
}

DataFrame is a wrapper for data frame, representing a distributed collection of data row.

type DataFrameReader

type DataFrameReader interface {
	// Format specifies data format (data source type) for the underlying data, e.g. parquet.
	Format(source string) DataFrameReader
	// Load reads the underlying data and returns a data frame.
	Load(path string) (DataFrame, error)
}

DataFrameReader supports reading data from storage and returning a data frame. TODO needs to implement other methods like Option(), Schema(), and also "strong typed" reading (e.g. Parquet(), Orc(), Csv(), etc.

type DataFrameWriter

type DataFrameWriter interface {
	// Mode specifies saving mode for the data, e.g. Append, Overwrite, ErrorIfExists.
	Mode(saveMode string) DataFrameWriter
	// Format specifies data format (data source type) for the underlying data, e.g. parquet.
	Format(source string) DataFrameWriter
	// Save writes data frame to the given path.
	Save(path string) error
}

DataFrameWriter supports writing data frame to storage.

type DataType

type DataType interface {
	TypeName() string
}

type DateType

type DateType struct {
}

func (DateType) TypeName

func (t DateType) TypeName() string

type DecimalType

type DecimalType struct {
}

func (DecimalType) TypeName

func (t DecimalType) TypeName() string

type DoubleType

type DoubleType struct {
}

func (DoubleType) TypeName

func (t DoubleType) TypeName() string

type FloatType

type FloatType struct {
}

func (FloatType) TypeName

func (t FloatType) TypeName() string

type GenericRowWithSchema

type GenericRowWithSchema struct {
	// contains filtered or unexported fields
}

func (*GenericRowWithSchema) Schema

func (r *GenericRowWithSchema) Schema() (*StructType, error)

func (*GenericRowWithSchema) Values

func (r *GenericRowWithSchema) Values() ([]any, error)

type IntegerType

type IntegerType struct {
}

func (IntegerType) TypeName

func (t IntegerType) TypeName() string

type LongType

type LongType struct {
}

func (LongType) TypeName

func (t LongType) TypeName() string

type RangePartitionColumn

type RangePartitionColumn struct {
	Name       string
	Descending bool
}

type Row

type Row interface {
	Schema() (*StructType, error)
	Values() ([]any, error)
}

type ShortType

type ShortType struct {
}

func (ShortType) TypeName

func (t ShortType) TypeName() string

type SparkSessionBuilder

type SparkSessionBuilder struct {
	// contains filtered or unexported fields
}

func (SparkSessionBuilder) Build

func (s SparkSessionBuilder) Build() (sparkSession, error)

func (SparkSessionBuilder) Remote

func (s SparkSessionBuilder) Remote(connectionString string) SparkSessionBuilder

type StringType

type StringType struct {
}

func (StringType) TypeName

func (t StringType) TypeName() string

type StructField

type StructField struct {
	Name     string
	DataType DataType
	Nullable bool // default should be true
}

type StructType

type StructType struct {
	TypeName string
	Fields   []StructField
}

type TimestampNtzType

type TimestampNtzType struct {
}

func (TimestampNtzType) TypeName

func (t TimestampNtzType) TypeName() string

type TimestampType

type TimestampType struct {
}

func (TimestampType) TypeName

func (t TimestampType) TypeName() string

type UnsupportedType

type UnsupportedType struct {
	TypeInfo any
}

func (UnsupportedType) TypeName

func (t UnsupportedType) TypeName() string