builtin

Documentation

Index

• Constants
• func DataProxyContentToFlightStreamBinary(data *datamesh.DomainData, r io.Reader, w utils.RecordWriter, ...) error
• func DataProxyContentToFlightStreamCSV(data *datamesh.DomainData, r io.Reader, w utils.RecordWriter) error
• func FlightStreamToDataProxyContentBinary(data *datamesh.DomainData, w io.Writer, reader *flight.Reader) error
• func FlightStreamToDataProxyContentCSV(data *datamesh.DomainData, w io.Writer, reader *flight.Reader) error
• type BuiltinLocalFileIO
  • func (fio *BuiltinLocalFileIO) GetEndpointURI() string
  • func (fio *BuiltinLocalFileIO) Read(ctx context.Context, rc *utils.DataMeshRequestContext, w utils.RecordWriter) error
  • func (fio *BuiltinLocalFileIO) Write(ctx context.Context, rc *utils.DataMeshRequestContext, reader *flight.Reader) error
• type BuiltinMySQLIO
  • func (o *BuiltinMySQLIO) GetEndpointURI() string
  • func (o *BuiltinMySQLIO) Read(ctx context.Context, rc *utils.DataMeshRequestContext, w utils.RecordWriter) error
  • func (o *BuiltinMySQLIO) Write(ctx context.Context, rc *utils.DataMeshRequestContext, stream *flight.Reader) error
• type BuiltinOssIO
  • func (o *BuiltinOssIO) GetEndpointURI() string
  • func (o *BuiltinOssIO) Read(ctx context.Context, rc *utils.DataMeshRequestContext, w utils.RecordWriter) error
  • func (o *BuiltinOssIO) Write(ctx context.Context, rc *utils.DataMeshRequestContext, reader *flight.Reader) error
• type DataMeshDataIOInterface
  • func NewBuiltinDirectIO(config *config.DataProxyConfig) DataMeshDataIOInterface
  • func NewBuiltinLocalFileIOChannel() DataMeshDataIOInterface
  • func NewBuiltinMySQLIOChannel() DataMeshDataIOInterface
  • func NewBuiltinOssIOChannel() DataMeshDataIOInterface
• type IOServer
  • func NewIOServer() *IOServer
  • func (d *IOServer) DoGet(tkt *flight.Ticket, fs flight.FlightService_DoGetServer) (err error)
  • func (d *IOServer) DoPut(stream flight.FlightService_DoPutServer) (err error)
  • func (d *IOServer) GetFlightInfo(ctx context.Context, reqCtx *utils.DataMeshRequestContext) (flightInfo *flight.FlightInfo, err error)
• type MySQLDownloader
  • func NewMySQLDownloader(ctx context.Context, db *sql.DB, data *datamesh.DomainData, ...) *MySQLDownloader
  • func (d *MySQLDownloader) DataProxyContentToFlightStreamSQL(w utils.RecordWriter) (err error)
• type MySQLUploader
  • func NewMySQLUploader(ctx context.Context, db *sql.DB, data *datamesh.DomainData, ...) *MySQLUploader
  • func (u *MySQLUploader) ArrowDataTypeToMySQLType(colType arrow.DataType) string
  • func (u *MySQLUploader) FlightStreamToDataProxyContentMySQL(reader *flight.Reader) (err error)
  • func (u *MySQLUploader) PrepareOutputTable(tableName string, columnNames []string, fields []arrow.Field) error
• type OSSUploader
  • func NewOSSUploader(ctx context.Context, client *s3.S3, bucket, objectKey string, ...) *OSSUploader
  • func (ow *OSSUploader) Close() error
  • func (ow *OSSUploader) FinishUpload() error
  • func (ow *OSSUploader) Write(p []byte) (int, error)

Constants

const CSVDefaultNullValue = "NULL"

const (
	ER_NO_SUCH_TABLE uint16 = 1146
)

define some mysql error

Functions

func DataProxyContentToFlightStreamBinary

func DataProxyContentToFlightStreamBinary(data *datamesh.DomainData, r io.Reader, w utils.RecordWriter, batchReadSize int) error

DataFlow(Binary): RemoteStorage(FileSystem/OSS/...) --> DataProxy --> Client

func DataProxyContentToFlightStreamCSV

func DataProxyContentToFlightStreamCSV(data *datamesh.DomainData, r io.Reader, w utils.RecordWriter) error

DataFlow(Table): RemoteStorage(FileSystem/OSS/...) --> DataProxy --> Client

func FlightStreamToDataProxyContentBinary

func FlightStreamToDataProxyContentBinary(data *datamesh.DomainData, w io.Writer, reader *flight.Reader) error

DataFlow(Binary): Client --> DataProxy --> RemoteStorage(FileSystem/OSS/...)

func FlightStreamToDataProxyContentCSV

func FlightStreamToDataProxyContentCSV(data *datamesh.DomainData, w io.Writer, reader *flight.Reader) error

DataFlow(Table): Client --> DataProxy --> RemoteStorage(FileSystem/OSS/...)

Types

type BuiltinLocalFileIO

type BuiltinLocalFileIO struct {
	// contains filtered or unexported fields
}

func (*BuiltinLocalFileIO) GetEndpointURI

func (fio *BuiltinLocalFileIO) GetEndpointURI() string

func (*BuiltinLocalFileIO) Read

func (fio *BuiltinLocalFileIO) Read(ctx context.Context, rc *utils.DataMeshRequestContext, w utils.RecordWriter) error

DataFlow: RemoteStorage(FileSystem/OSS/...) --> DataProxy --> Client

func (*BuiltinLocalFileIO) Write

func (fio *BuiltinLocalFileIO) Write(ctx context.Context, rc *utils.DataMeshRequestContext, reader *flight.Reader) error

DataFlow: Client --> DataProxy --> RemoteStorage(FileSystem/OSS/...)

type BuiltinMySQLIO

type BuiltinMySQLIO struct {
	// contains filtered or unexported fields
}

BuiltinMySQLIO defined the MySQL read & write methond

func (*BuiltinMySQLIO) GetEndpointURI

func (o *BuiltinMySQLIO) GetEndpointURI() string

func (*BuiltinMySQLIO) Read

func (o *BuiltinMySQLIO) Read(ctx context.Context, rc *utils.DataMeshRequestContext, w utils.RecordWriter) error

DataFlow: RemoteStorage(MySQL/...) --> DataProxy --> Client

func (*BuiltinMySQLIO) Write

func (o *BuiltinMySQLIO) Write(ctx context.Context, rc *utils.DataMeshRequestContext, stream *flight.Reader) error

DataFlow: Client --> DataProxy --> RemoteStorage(MySQL/...)

type BuiltinOssIO

type BuiltinOssIO struct {
	// contains filtered or unexported fields
}

BuiltinOssIO defined the oss read & write methond

func (*BuiltinOssIO) GetEndpointURI

func (o *BuiltinOssIO) GetEndpointURI() string

func (*BuiltinOssIO) Read

func (o *BuiltinOssIO) Read(ctx context.Context, rc *utils.DataMeshRequestContext, w utils.RecordWriter) error

DataFlow: RemoteStorage(FileSystem/OSS/...) --> DataProxy --> Client

func (*BuiltinOssIO) Write

func (o *BuiltinOssIO) Write(ctx context.Context, rc *utils.DataMeshRequestContext, reader *flight.Reader) error

DataFlow: Client --> DataProxy --> RemoteStorage(FileSystem/OSS/...)

type DataMeshDataIOInterface

type DataMeshDataIOInterface interface {
	Read(ctx context.Context, rc *utils.DataMeshRequestContext, w utils.RecordWriter) error

	Write(ctx context.Context, rc *utils.DataMeshRequestContext, stream *flight.Reader) error

	GetEndpointURI() string
}

func NewBuiltinDirectIO

func NewBuiltinDirectIO(config *config.DataProxyConfig) DataMeshDataIOInterface

func NewBuiltinLocalFileIOChannel

func NewBuiltinLocalFileIOChannel() DataMeshDataIOInterface

func NewBuiltinMySQLIOChannel

func NewBuiltinMySQLIOChannel() DataMeshDataIOInterface

func NewBuiltinOssIOChannel

func NewBuiltinOssIOChannel() DataMeshDataIOInterface

type IOServer

type IOServer struct {
	// contains filtered or unexported fields
}

func NewIOServer

func NewIOServer() *IOServer

func (*IOServer) DoGet

func (d *IOServer) DoGet(tkt *flight.Ticket, fs flight.FlightService_DoGetServer) (err error)

func (*IOServer) DoPut

func (d *IOServer) DoPut(stream flight.FlightService_DoPutServer) (err error)

func (*IOServer) GetFlightInfo

func (d *IOServer) GetFlightInfo(ctx context.Context, reqCtx *utils.DataMeshRequestContext) (flightInfo *flight.FlightInfo, err error)

type MySQLDownloader

type MySQLDownloader struct {
	// contains filtered or unexported fields
}

func NewMySQLDownloader

func NewMySQLDownloader(ctx context.Context, db *sql.DB, data *datamesh.DomainData, query *datamesh.CommandDomainDataQuery) *MySQLDownloader

func (*MySQLDownloader) DataProxyContentToFlightStreamSQL

func (d *MySQLDownloader) DataProxyContentToFlightStreamSQL(w utils.RecordWriter) (err error)

type MySQLUploader

type MySQLUploader struct {
	// contains filtered or unexported fields
}

func NewMySQLUploader

func NewMySQLUploader(ctx context.Context, db *sql.DB, data *datamesh.DomainData, query *datamesh.CommandDomainDataQuery) *MySQLUploader

func (*MySQLUploader) ArrowDataTypeToMySQLType

func (u *MySQLUploader) ArrowDataTypeToMySQLType(colType arrow.DataType) string

func (*MySQLUploader) FlightStreamToDataProxyContentMySQL

func (u *MySQLUploader) FlightStreamToDataProxyContentMySQL(reader *flight.Reader) (err error)

func (*MySQLUploader) PrepareOutputTable

func (u *MySQLUploader) PrepareOutputTable(tableName string, columnNames []string, fields []arrow.Field) error

type OSSUploader

type OSSUploader struct {
	// contains filtered or unexported fields
}

func NewOSSUploader

func NewOSSUploader(ctx context.Context, client *s3.S3, bucket, objectKey string, partSizeBytes int) *OSSUploader

func (*OSSUploader) Close

func (ow *OSSUploader) Close() error

func (*OSSUploader) FinishUpload

func (ow *OSSUploader) FinishUpload() error

func (*OSSUploader) Write

func (ow *OSSUploader) Write(p []byte) (int, error)