mvdata

package
v0.40.4 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: May 19, 2022 License: Apache-2.0 Imports: 39 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var ErrProvidedPkNotFound = errors.New("provided primary key not found")

Functions

func InferSchema 

func InferSchema(ctx context.Context, root *doltdb.RootValue, rd table.TableReadCloser, tableName string, pks []string, args actions.InferenceArgs) (schema.Schema, error)

func NewSqlEngineReader 

func NewSqlEngineReader(ctx context.Context, dEnv *env.DoltEnv, tableName string) (*sqlEngineTableReader, error)

func NewSqlEngineTableReaderWithEngine 

func NewSqlEngineTableReaderWithEngine(sqlCtx *sql.Context, se *sqle.Engine, db dsqle.Database, root *doltdb.RootValue, tableName string) (*sqlEngineTableReader, error)

Used by Dolthub API

func SchAndTableNameFromFile 

func SchAndTableNameFromFile(ctx context.Context, path string, fs filesys.ReadableFS, root *doltdb.RootValue) (string, schema.Schema, error)

SchAndTableNameFromFile reads a SQL schema file and creates a Dolt schema from it.

Types

type ChannelRowSource 

type ChannelRowSource struct {
	// contains filtered or unexported fields
}

ChannelRowSource is a sql.Node that wraps a channel as a sql.RowIter.

func NewChannelRowSource 

func NewChannelRowSource(schema sql.Schema, rowChannel chan sql.Row) *ChannelRowSource

NewChannelRowSource returns a ChannelRowSource object.

func (*ChannelRowSource) CheckPrivileges 

func (c *ChannelRowSource) CheckPrivileges(ctx *sql.Context, opChecker sql.PrivilegedOperationChecker) bool

CheckPrivileges implements the sql.Node interface.

func (*ChannelRowSource) Children 

func (c *ChannelRowSource) Children() []sql.Node

Children implements the sql.Node interface.

func (*ChannelRowSource) Resolved 

func (c *ChannelRowSource) Resolved() bool

Resolved implements the sql.Node interface.

func (*ChannelRowSource) RowIter 

func (c *ChannelRowSource) RowIter(ctx *sql.Context, row sql.Row) (sql.RowIter, error)

RowIter implements the sql.Node interface.

func (*ChannelRowSource) Schema 

func (c *ChannelRowSource) Schema() sql.Schema

Schema implements the sql.Node interface.

func (*ChannelRowSource) String 

func (c *ChannelRowSource) String() string

String implements the sql.Node interface.

func (*ChannelRowSource) WithChildren 

func (c *ChannelRowSource) WithChildren(children ...sql.Node) (sql.Node, error)

WithChildren implements the sql.Node interface.

type CsvOptions 

type CsvOptions struct {
	Delim string
}

type DataFormat 

type DataFormat string

DataFormat is an enumeration of the valid data formats 

const (
	// InvalidDataFormat is the format of a data lotacion that isn't valid
	InvalidDataFormat DataFormat = "invalid"

	// DoltDB is the format of a data location for a dolt table
	DoltDB DataFormat = "doltdb"

	// CsvFile is the format of a data location that is a .csv file
	CsvFile DataFormat = ".csv"

	// PsvFile is the format of a data location that is a .psv file
	PsvFile DataFormat = ".psv"

	// XlsxFile is the format of a data location that is a .xlsx file
	XlsxFile DataFormat = ".xlsx"

	// JsonFile is the format of a data location that is a json file
	JsonFile DataFormat = ".json"

	// SqlFile is the format of a data location that is a .sql file
	SqlFile DataFormat = ".sql"

	// ParquetFile is the format of a data location that is a .paquet file
	ParquetFile DataFormat = ".parquet"
)

func DFFromString 

func DFFromString(dfStr string) DataFormat

DFFromString returns a data object from a string.

func (DataFormat) ReadableStr 

func (df DataFormat) ReadableStr() string

ReadableStr returns a human readable string for a DataFormat

type DataLocation 

type DataLocation interface {
	fmt.Stringer

	// Exists returns true if the DataLocation already exists
	Exists(ctx context.Context, root *doltdb.RootValue, fs filesys.ReadableFS) (bool, error)

	// NewReader creates a TableReadCloser for the DataLocation
	NewReader(ctx context.Context, root *doltdb.RootValue, fs filesys.ReadableFS, opts interface{}) (rdCl table.SqlRowReader, sorted bool, err error)

	// NewCreatingWriter will create a TableWriteCloser for a DataLocation that will create a new table, or overwrite
	// an existing table.
	NewCreatingWriter(ctx context.Context, mvOpts DataMoverOptions, root *doltdb.RootValue, outSch schema.Schema, opts editor.Options, wr io.WriteCloser) (table.SqlTableWriter, error)
}

DataLocation is an interface that can be used to read or write from the source or the destination of a move operation.

func NewDataLocation 

func NewDataLocation(path, fileFmtStr string) DataLocation

NewDataLocation creates a DataLocation object from a path and a format string. If the path is the name of a table then a TableDataLocation will be returned. If the path is empty a StreamDataLocation is returned. Otherwise a FileDataLocation is returned. For FileDataLocations and StreamDataLocations, if a file format is provided explicitly then it is used as the format, otherwise, when it can be, it is inferred from the path for files. Inference is based on the file's extension.

type DataMover 

type DataMover struct {
	Rd         table.TableReadCloser
	Transforms *pipeline.TransformCollection
	Wr         table.TableWriteCloser
	ContOnErr  bool
}

type DataMoverCloser 

type DataMoverCloser interface {
	table.TableWriteCloser
	Flush(context.Context) (*doltdb.RootValue, error)
}

type DataMoverCreationErrType 

type DataMoverCreationErrType string
const (
	CreateReaderErr   DataMoverCreationErrType = "Create reader error"
	NomsKindSchemaErr DataMoverCreationErrType = "Invalid schema error"
	SchemaErr         DataMoverCreationErrType = "Schema error"
	MappingErr        DataMoverCreationErrType = "Mapping error"
	ReplacingErr      DataMoverCreationErrType = "Replacing error"
	CreateMapperErr   DataMoverCreationErrType = "Mapper creation error"
	CreateWriterErr   DataMoverCreationErrType = "Create writer error"
	CreateSorterErr   DataMoverCreationErrType = "Create sorter error"
)

type DataMoverCreationError 

type DataMoverCreationError struct {
	ErrType DataMoverCreationErrType
	Cause   error
}

func (*DataMoverCreationError) String 

func (dmce *DataMoverCreationError) String() string

type DataMoverOptions 

type DataMoverOptions interface {
	IsBatched() bool
	WritesToTable() bool
	SrcName() string
	DestName() string
}

type DataMoverPipeline 

type DataMoverPipeline struct {
	// contains filtered or unexported fields
}

DataMoverPipeline is an errgroup based pipeline that reads rows from a reader and writes them to a destination with a writer.

func NewDataMoverPipeline 

func NewDataMoverPipeline(ctx context.Context, rd table.SqlRowReader, wr table.SqlTableWriter) *DataMoverPipeline

func (*DataMoverPipeline) Execute 

func (e *DataMoverPipeline) Execute() error

type FileDataLocation 

type FileDataLocation struct {
	// Path is the path of the file on the filesystem
	Path string

	// Format is the DataFormat of the file
	Format DataFormat
}

FileDataLocation is a file that that can be imported from or exported to.

func (FileDataLocation) Exists 

func (dl FileDataLocation) Exists(ctx context.Context, root *doltdb.RootValue, fs filesys.ReadableFS) (bool, error)

Exists returns true if the DataLocation already exists

func (FileDataLocation) NewCreatingWriter 

func (dl FileDataLocation) NewCreatingWriter(ctx context.Context, mvOpts DataMoverOptions, root *doltdb.RootValue, outSch schema.Schema, opts editor.Options, wr io.WriteCloser) (table.SqlTableWriter, error)

NewCreatingWriter will create a TableWriteCloser for a DataLocation that will create a new table, or overwrite an existing table.

func (FileDataLocation) NewReader 

func (dl FileDataLocation) NewReader(ctx context.Context, root *doltdb.RootValue, fs filesys.ReadableFS, opts interface{}) (rdCl table.SqlRowReader, sorted bool, err error)

NewReader creates a TableReadCloser for the DataLocation

func (FileDataLocation) String 

func (dl FileDataLocation) String() string

String returns a string representation of the data location.

type JSONOptions 

type JSONOptions struct {
	TableName string
	SchFile   string
}

type MoverOptions 

type MoverOptions struct {
	ContinueOnErr  bool
	Force          bool
	TableToWriteTo string
	Operation      TableImportOp
	DisableFks     bool
}

type ParquetOptions 

type ParquetOptions struct {
	TableName string
	SchFile   string
}

type SqlEngineTableWriter 

type SqlEngineTableWriter struct {
	// contains filtered or unexported fields
}

SqlEngineTableWriter is a utility for importing a set of rows through the sql engine.

func NewSqlEngineTableWriter 

func NewSqlEngineTableWriter(ctx context.Context, dEnv *env.DoltEnv, createTableSchema, rowOperationSchema schema.Schema, options *MoverOptions, statsCB noms.StatsCB) (*SqlEngineTableWriter, error)

func NewSqlEngineTableWriterWithEngine 

func NewSqlEngineTableWriterWithEngine(ctx *sql.Context, eng *sqle.Engine, db dsqle.Database, createTableSchema, rowOperationSchema schema.Schema, options *MoverOptions, statsCB noms.StatsCB) (*SqlEngineTableWriter, error)

Used by Dolthub API

func (*SqlEngineTableWriter) Commit 

func (s *SqlEngineTableWriter) Commit(ctx context.Context) error

func (*SqlEngineTableWriter) RowOperationSchema 

func (s *SqlEngineTableWriter) RowOperationSchema() sql.PrimaryKeySchema

func (*SqlEngineTableWriter) TableSchema 

func (s *SqlEngineTableWriter) TableSchema() sql.PrimaryKeySchema

func (*SqlEngineTableWriter) WriteRows 

func (s *SqlEngineTableWriter) WriteRows(ctx context.Context, inputChannel chan sql.Row, badRowCb func(*pipeline.TransformRowFailure) bool) (err error)

type StreamDataLocation 

type StreamDataLocation struct {
	Format DataFormat
	Writer io.WriteCloser
	Reader io.ReadCloser
}

StreamDataLocation is a process stream that that can be imported from or exported to.

func (StreamDataLocation) Exists 

func (dl StreamDataLocation) Exists(ctx context.Context, root *doltdb.RootValue, fs filesys.ReadableFS) (bool, error)

Exists returns true if the DataLocation already exists

func (StreamDataLocation) NewCreatingWriter 

func (dl StreamDataLocation) NewCreatingWriter(ctx context.Context, mvOpts DataMoverOptions, root *doltdb.RootValue, outSch schema.Schema, opts editor.Options, wr io.WriteCloser) (table.SqlTableWriter, error)

NewCreatingWriter will create a TableWriteCloser for a DataLocation that will create a new table, or overwrite an existing table.

func (StreamDataLocation) NewReader 

func (dl StreamDataLocation) NewReader(ctx context.Context, root *doltdb.RootValue, fs filesys.ReadableFS, opts interface{}) (rdCl table.SqlRowReader, sorted bool, err error)

NewReader creates a TableReadCloser for the DataLocation

func (StreamDataLocation) String 

func (dl StreamDataLocation) String() string

String returns a string representation of the data location.

type TableImportOp 

type TableImportOp string
const (
	CreateOp  TableImportOp = "overwrite"
	ReplaceOp TableImportOp = "replace"
	UpdateOp  TableImportOp = "update"
)

type XlsxOptions 

type XlsxOptions struct {
	SheetName string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.