database

Feb 3, 2024

Changes in this version

+ type StarRocksConn struct

+ URL string

+ func (conn *StarRocksConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *StarRocksConn) GenerateDDL(tableFName string, data iop.Dataset, temporary bool) (string, error)

+ func (conn *StarRocksConn) GetURL(newURL ...string) string

+ func (conn *StarRocksConn) Init() error

+ func (conn *StarRocksConn) InsertBatchStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *StarRocksConn) StreamLoad(feURL, tableFName string, df *iop.Dataflow) (count uint64, err error)

Jan 10, 2024

Changes in this version

type ClickhouseConn

+ func (conn *ClickhouseConn) ConnString() string

Sep 28, 2023

Changes in this version

type SQLiteConn

+ func (conn *SQLiteConn) GetSchemata(schemaName string, tableNames ...string) (Schemata, error)

Sep 26, 2023

Changes in this version

+ func HasVariedCase(text string) bool

Sep 23, 2023

Changes in this version

type Pool

+ DuckDbs map[string]*DuckDbConn

Aug 23, 2023

Changes in this version

+ var DuckDbFileContext = map[string]*g.Context

Aug 22, 2023

Changes in this version

+ var DuckDbUseTempFile = false

Jun 20, 2023

Changes in this version

type DataAnalyzer

+ func (da *DataAnalyzer) ProcessRelationsInteger() (err error)

+ func (da *DataAnalyzer) ProcessRelationsString() (err error)

+ func (da *DataAnalyzer) WriteRelationsYaml(path string) (err error)

May 15, 2023

Changes in this version

type BigQueryConn

+ func (conn *BigQueryConn) CastColumnForSelect(srcCol iop.Column, tgtCol iop.Column) (selectStr string)

May 3, 2023

Changes in this version

+ func TableExists(conn Connection, tableFName string) (exists bool, err error)

Feb 24, 2023

Changes in this version

+ const DuckDbVersion

+ const SQLiteVersion

+ func EnsureBinDuckDB() (binPath string, err error)

+ func EnsureBinSQLite() (binPath string, err error)

type DuckDbConn

+ func (conn *DuckDbConn) Close() error

+ func (conn *DuckDbConn) Connect(timeOut ...int) (err error)

+ func (conn *DuckDbConn) ExecContext(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error)

+ func (conn *DuckDbConn) ExecMultiContext(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error)

+ func (conn *DuckDbConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

+ func (conn *DuckDbConn) InsertBatchStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *DuckDbConn) InsertStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *DuckDbConn) StreamRowsContext(ctx context.Context, sql string, options ...map[string]interface{}) (ds *iop.Datastream, err error)

Feb 18, 2023

Changes in this version

type MsSQLServerConn

+ func (conn *MsSQLServerConn) BcpImportFile(tableFName, filePath string) (count uint64, err error)

+ func (conn *MsSQLServerConn) BcpImportFileParrallel(tableFName string, ds *iop.Datastream) (count uint64, err error)

Feb 17, 2023

Changes in this version

type Table

+ func (t *Table) Select(fields ...string) string

Feb 17, 2023

Changes in this version

type DuckDbConn — darwin/amd64, js/wasm, linux/amd64

+ func (conn *DuckDbConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

Feb 17, 2023

Changes in this version

type SQLiteConn

+ func (conn *SQLiteConn) BulkImportStreamBatch(table Table, batchR *iop.BatchReader) (err error)

Feb 16, 2023

Changes in this version

type DuckDbConn — windows/amd64

+ func (conn *DuckDbConn) Close() error

+ func (conn *DuckDbConn) ExecContext(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error)

+ func (conn *DuckDbConn) ExecMultiContext(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error)

+ func (conn *DuckDbConn) InsertBatchStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *DuckDbConn) InsertStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *DuckDbConn) StreamRowsContext(ctx context.Context, sql string, options ...map[string]interface{}) (ds *iop.Datastream, err error)

Feb 16, 2023

Changes in this version

type SQLiteConn

+ func (conn *SQLiteConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

Feb 15, 2023

Changes in this version

type SnowflakeConn

+ func (conn *SnowflakeConn) GetViews(schema string) (data iop.Dataset, err error)

Feb 14, 2023

Changes in this version

type BaseTransaction

+ func (t *BaseTransaction) Connection() Connection

type BlankTransaction

+ func (t *BlankTransaction) Connection() Connection

type Transaction

+ Connection func() Connection

Feb 11, 2023

Changes in this version

+ type DuckDbConn struct

+ URL string

+ func (conn *DuckDbConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *DuckDbConn) GetURL(newURL ...string) string

+ func (conn *DuckDbConn) Init() error

Feb 1, 2023

Changes in this version

+ func ParseColumnName(text string, dialect dbio.Type) (colName string, err error)

Dec 16, 2022

Changes in this version

type Schemata

+ func GetTablesSchemata(conn Connection, tableNames ...string) (schemata Schemata, err error)

Dec 2, 2022

Changes in this version

type Table

+ func (t *Table) DatabaseQ() string

+ func (t *Table) NameQ() string

+ func (t *Table) SchemaQ() string

Nov 23, 2022

Changes in this version

type BigTableConn

+ func (conn *BigTableConn) GetColumnsFull(tableFName string) (iop.Dataset, error)

+ func (conn *BigTableConn) GetSchemas() (iop.Dataset, error)

Nov 17, 2022

Changes in this version

+ func NativeTypeToGeneral(name, dbType string, conn Connection) (colType iop.ColumnType)

Nov 15, 2022

Changes in this version

type BigQueryConn

+ func (conn *BigQueryConn) ExecMultiContext(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error)

Nov 15, 2022

Changes in this version

+ func ChangeColumnTypeViaAdd(conn Connection, table Table, col iop.Column) (err error)

Oct 23, 2022

Changes in this version

type BigTableConn

+ func (conn *BigTableConn) GetSchemata(schemaName, tableName string) (schemata Schemata, err error)

Oct 23, 2022

Changes in this version

+ type BigTableAction string

+ const BTCreateColumnFamily

+ const BTCreateTable

+ const BTDeleteTable

+ const BTTableInfo

+ type BigTableConn struct

+ Client *bigtable.Client

+ InstanceID string

+ Location string

+ ProjectID string

+ URL string

+ func (conn *BigTableConn) BulkExportFlow(tableNames ...string) (df *iop.Dataflow, err error)

+ func (conn *BigTableConn) Close() error

+ func (conn *BigTableConn) Connect(timeOut ...int) error

+ func (conn *BigTableConn) ExecContext(ctx context.Context, payload string, args ...interface{}) (result sql.Result, err error)

+ func (conn *BigTableConn) GetColumns(tableFName string, fields ...string) (columns iop.Columns, err error)

+ func (conn *BigTableConn) GetSQLColumns(sqls ...string) (columns iop.Columns, err error)

+ func (conn *BigTableConn) GetTables(schema string) (data iop.Dataset, err error)

+ func (conn *BigTableConn) GetViews(schema string) (data iop.Dataset, err error)

+ func (conn *BigTableConn) Init() error

+ func (conn *BigTableConn) InsertBatchStream(table string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *BigTableConn) NewTransaction(ctx context.Context, options ...*sql.TxOptions) (tx Transaction, err error)

+ func (conn *BigTableConn) StreamRowsContext(ctx context.Context, table string, options ...map[string]interface{}) (ds *iop.Datastream, err error)

+ type BigTableQuery struct

+ Action BigTableAction

+ ColumnFamilies []string

+ Table string

Oct 11, 2022

Changes in this version

type BigQueryConn

+ func (conn *BigQueryConn) ExportToGCS(sql string, gcsURI string) error

Oct 9, 2022

Changes in this version

type BigQueryConn

+ func (conn *BigQueryConn) CopyFromLocal(localURI string, tableFName string, dsColumns []iop.Column) error

Oct 8, 2022

Changes in this version

type BigQueryConn

+ func (conn *BigQueryConn) LoadCSVFromReader(tableFName string, reader io.Reader, dsColumns []iop.Column) error

type OracleConn

+ func (conn *OracleConn) ExecMultiContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

Oct 5, 2022

Changes in this version

type ClickhouseConn

+ func (conn *ClickhouseConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *ClickhouseConn) GenerateInsertStatement(tableName string, fields []string, numRows int) string

Sep 30, 2022

Changes in this version

type SQLiteConn

+ func (conn *SQLiteConn) GetURL(newURL ...string) string

Sep 29, 2022

Changes in this version

+ func GenerateAlterDDL(conn Connection, table Table, newColumns iop.Columns) (bool, error)

+ func GetQualifierQuote(dialect dbio.Type) string

type BaseConn

+ func (conn *BaseConn) GetTableColumns(table *Table, fields ...string) (columns iop.Columns, err error)

type Connection

+ func Clone(conn Connection) (newConn Connection, err error)

+ GetTableColumns func(table *Table, fields ...string) (columns iop.Columns, err error)

type PostgresConn

+ func (conn *PostgresConn) BulkImportStream0(tableFName string, ds *iop.Datastream) (count uint64, err error)

type Table

+ Dialect dbio.Type

+ SQL string

+ func ParseTableName(text string, dialect dbio.Type) (table Table, err error)

+ func (t *Table) FDQN() string

+ func (t *Table) IsQuery() bool

Sep 22, 2022

Changes in this version

+ var InferDBStream = false

Aug 19, 2022

Changes in this version

type ClickhouseConn

+ func (conn *ClickhouseConn) NewTransaction(ctx context.Context, options ...*sql.TxOptions) (tx Transaction, err error)

+ type ColumnType struct

+ DatabaseTypeName string

+ Length int

+ Name string

+ Nullable bool

+ Precision int

+ Scale int

+ Sourced bool

Aug 19, 2022

Changes in this version

+ type BaseTransaction struct

+ Conn Connection

+ Tx *sqlx.Tx

+ func (t *BaseTransaction) Commit() (err error)

+ func (t *BaseTransaction) Context() *g.Context

+ func (t *BaseTransaction) DisableTrigger(tableName, triggerName string) (err error)

+ func (t *BaseTransaction) EnableTrigger(tableName, triggerName string) (err error)

+ func (t *BaseTransaction) Exec(sql string, args ...interface{}) (result sql.Result, err error)

+ func (t *BaseTransaction) ExecContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (t *BaseTransaction) ExecMultiContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (t *BaseTransaction) InsertBatchStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (t *BaseTransaction) InsertStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (t *BaseTransaction) Prepare(query string) (stmt *sql.Stmt, err error)

+ func (t *BaseTransaction) QueryContext(ctx context.Context, q string, args ...interface{}) (result *sqlx.Rows, err error)

+ func (t *BaseTransaction) Rollback() (err error)

+ func (t *BaseTransaction) Upsert(sourceTable, targetTable string, pkFields []string) (count uint64, err error)

+ func (t *BaseTransaction) UpsertStream(tableFName string, ds *iop.Datastream, pk []string) (count uint64, err error)

type BigQueryConn

+ func (conn *BigQueryConn) NewTransaction(ctx context.Context, options ...*sql.TxOptions) (tx Transaction, err error)

+ type BlankTransaction struct

+ Conn Connection

+ func (t *BlankTransaction) Commit() (err error)

+ func (t *BlankTransaction) Context() *g.Context

+ func (t *BlankTransaction) ExecContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (t *BlankTransaction) ExecMultiContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (t *BlankTransaction) Prepare(query string) (stmt *sql.Stmt, err error)

+ func (t *BlankTransaction) QueryContext(ctx context.Context, q string, args ...interface{}) (result *sqlx.Rows, err error)

+ func (t *BlankTransaction) Rollback() (err error)

+ type ManualTransaction struct

+ Conn Connection

+ func (t *ManualTransaction) Commit() (err error)

+ func (t *ManualTransaction) Context() *g.Context

+ func (t *ManualTransaction) ExecContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (t *ManualTransaction) ExecMultiContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (t *ManualTransaction) Prepare(query string) (stmt *sql.Stmt, err error)

+ func (t *ManualTransaction) QueryContext(ctx context.Context, q string, args ...interface{}) (result *sqlx.Rows, err error)

+ func (t *ManualTransaction) Rollback() (err error)

Aug 5, 2022

Changes in this version

+ func AddMissingColumns(conn Connection, tableName string, newCols iop.Columns) (err error)

Aug 2, 2022

Changes in this version

+ const RelationManyToMany

+ const RelationManyToOne

+ const RelationOneToMany

+ const RelationOneToOne

type Column

+ Database string

+ GenType string

+ Schema string

+ Stats ColumnStats

+ Table string

+ func (c Column) IsUnique() bool

+ func (c Column) Key() string

+ type ColumnStats struct

+ DistinctCount int64

+ LengthMaximum int

+ LengthMinimun int

+ RowCount int64

+ ValueCount int64

+ func (cs *ColumnStats) DistinctPercent() float64

+ func (cs *ColumnStats) DuplicateCount() int64

+ func (cs *ColumnStats) DuplicatePercent() float64

+ func (cs *ColumnStats) NullCount() int64

+ func (cs *ColumnStats) NullPercent() float64

+ type DataAnalyzer struct

+ ColumnMap map[string]Column

+ Conn Connection

+ Options DataAnalyzerOptions

+ RelationMap map[string]map[string]Relation

+ Schemata Schemata

+ func NewDataAnalyzer(conn Connection, opts DataAnalyzerOptions) (da *DataAnalyzer, err error)

+ func (da *DataAnalyzer) AnalyzeColumns(sampleSize int) (err error)

+ func (da *DataAnalyzer) GetManyToMany(nonUniqueCols []Column) (err error)

+ func (da *DataAnalyzer) GetOneToMany(uniqueCols, nonUniqueCols []Column) (err error)

+ func (da *DataAnalyzer) GetOneToOne(uniqueCols []Column) (err error)

+ func (da *DataAnalyzer) GetSchemata(force bool) (err error)

+ func (da *DataAnalyzer) ProcessRelations() (err error)

+ type DataAnalyzerOptions struct

+ DbName string

+ SchemaNames []string

+ type Relation string

+ type StatFieldSQL struct

+ Name string

+ TemplateSQL string

Apr 4, 2022

Changes in this version

type Connection

+ SumbitTemplate func(level string, templateMap map[string]string, name string, ...) (data iop.Dataset, err error)

Mar 13, 2022

Changes in this version

type BigQueryConn

+ func (conn *BigQueryConn) GetDatabases() (iop.Dataset, error)

+ func (conn *BigQueryConn) GetSchemas() (iop.Dataset, error)

+ func (conn *BigQueryConn) GetSchemata(schemaName, tableName string) (Schemata, error)

Jan 27, 2022

Changes in this version

type BaseConn

+ func (conn *BaseConn) Info() (ci ConnInfo)

+ type ConnInfo struct

+ Database string

+ Host string

+ Password string

+ Port int

+ Role string

+ Schema string

+ URL *net.URL

+ User string

+ Warehouse string

type Connection

+ Info func() ConnInfo

Oct 15, 2021

Changes in this version

type BaseConn

+ func (conn *BaseConn) ConnString() string

+ func (conn *BaseConn) CurrentDatabase() (dbName string, err error)

+ func (conn *BaseConn) GetDatabases() (iop.Dataset, error)

type Connection

+ ConnString func() string

+ CurrentDatabase func() (string, error)

+ GetDatabases func() (iop.Dataset, error)

type RedshiftConn

+ func (conn *RedshiftConn) ConnString() string

type Schemata

+ func GetSchemataAll(conn Connection) (schemata Schemata, err error)

+ func (s *Schemata) LoadTablesJSON(payload string) error

type SnowflakeConn

+ func (conn *SnowflakeConn) ConnString() string

+ func (conn *SnowflakeConn) GetDatabases() (data iop.Dataset, err error)

type Table

+ Database string

Oct 14, 2021

Changes in this version

type SnowflakeConn

+ func (conn *SnowflakeConn) GetColumnsFull(tableFName string) (data iop.Dataset, err error)

Oct 8, 2021

Changes in this version

+ type Column struct

+ Name string

+ Position int

+ Type string

+ type Database struct

+ Name string

+ Schemas map[string]Schema

+ func (db *Database) Columns() map[string]*Column

+ func (db *Database) Tables() map[string]*Table

type Schema

+ func (schema *Schema) Columns() map[string]*Column

type Schemata

+ Databases map[string]Database

+ func (s *Schemata) Columns() map[string]*Column

+ func (s *Schemata) Database() Database

Jul 24, 2021

Changes in this version

type BaseConn

+ Log []string

+ func (conn *BaseConn) AddLog(text string)

+ func (conn *BaseConn) Base() *BaseConn

+ func (conn *BaseConn) GetSchemata(schemaName, tableName string) (Schemata, error)

+ func (conn *BaseConn) ProcessTemplate(level, text string, values map[string]interface{}) (sql string, err error)

+ func (conn *BaseConn) SumbitTemplate(level string, templateMap map[string]string, name string, ...) (data iop.Dataset, err error)

+ type ClickhouseConn struct

+ URL string

+ func (conn *ClickhouseConn) ExecContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (conn *ClickhouseConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

+ func (conn *ClickhouseConn) GetURL(newURL ...string) string

+ func (conn *ClickhouseConn) Init() error

type Connection

+ Base func() *BaseConn

+ GetSchemata func(string, string) (Schemata, error)

+ ProcessTemplate func(level, text string, values map[string]interface{}) (sql string, err error)

type SnowflakeConn

+ func (conn *SnowflakeConn) GetFile(internalStagePath, fPath string) (err error)

+ func (conn *SnowflakeConn) UnloadViaStage(sqls ...string) (df *iop.Dataflow, err error)

type Table

+ Schema string

May 31, 2021

Changes in this version

+ var Debug = false

+ func InsertBatchStream(conn Connection, tx *Transaction, tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func InsertStream(conn Connection, tx *Transaction, tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func PK(obj interface{}) (pk []string)

+ func ParseSQLMultiStatements(sql string) (sqls g.Strings)

+ func UID(obj interface{}) string

+ func Upsert(conn Connection, tx *Transaction, sourceTable, targetTable string, ...) (count int64, err error)

type BaseConn

+ func (conn *BaseConn) BeginContext(ctx context.Context, options ...*sql.TxOptions) (err error)

+ func (conn *BaseConn) CreateTable(tableName string, cols iop.Columns, tableDDL string) (err error)

+ func (conn *BaseConn) CreateTemporaryTable(tableName string, cols iop.Columns) (err error)

+ func (conn *BaseConn) DbX() *DbX

+ func (conn *BaseConn) ExecMulti(sql string, args ...interface{}) (result sql.Result, err error)

+ func (conn *BaseConn) ExecMultiContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (conn *BaseConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

+ func (conn *BaseConn) GetAnalysis(analysisName string, values map[string]interface{}) (sql string, err error)

+ func (conn *BaseConn) GetAnalysisField(analysisName string, tableFName string, fields ...string) (sql string, err error)

+ func (conn *BaseConn) GetAnalysisTable(analysisName string, tableFNames ...string) (sql string, err error)

+ func (conn *BaseConn) GetNativeType(col iop.Column) (nativeType string, err error)

+ func (conn *BaseConn) NewTransaction(ctx context.Context, options ...*sql.TxOptions) (*Transaction, error)

+ func (conn *BaseConn) SetTx(tx *Transaction)

type BigQueryConn

+ func (conn *BigQueryConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

type Connection

+ BeginContext func(ctx context.Context, options ...*sql.TxOptions) error

+ CreateTable func(tableName string, cols iop.Columns, tableDDL string) (err error)

+ CreateTemporaryTable func(tableName string, cols iop.Columns) (err error)

+ DbX func() *DbX

+ ExecMulti func(sql string, args ...interface{}) (result sql.Result, err error)

+ ExecMultiContext func(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error)

+ GenerateUpsertSQL func(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

+ GetAnalysis func(string, map[string]interface{}) (string, error)

+ GetAnalysisField func(string, string, ...string) (string, error)

+ GetAnalysisTable func(string, ...string) (string, error)

+ GetNativeType func(col iop.Column) (nativeType string, err error)

+ NewTransaction func(ctx context.Context, options ...*sql.TxOptions) (*Transaction, error)

+ Prepare func(query string) (stmt *sql.Stmt, err error)

+ SetTx func(*Transaction)

+ type DbX struct

+ func (x *DbX) Delete(o interface{}) (cnt int, err error)

+ func (x *DbX) Get(o interface{}, fields ...string) (err error)

+ func (x *DbX) Insert(o interface{}, fields ...string) (err error)

+ func (x *DbX) Select(o interface{}, fields ...string) (err error)

+ func (x *DbX) TableName(o interface{}) (name string)

+ func (x *DbX) Update(o interface{}, fields ...string) (cnt int, err error)

+ func (x *DbX) Upsert(o interface{}, fields ...string) (cnt int, err error)

+ func (x *DbX) Where(where ...interface{}) *DbX

+ type ModelDbX struct

+ Ptr interface{}

+ RowsAffected int

+ func (m *ModelDbX) Bind(bindFunc func(p interface{}) error, objPtr interface{}) (err error)

+ func (m *ModelDbX) Delete(db *sqlx.DB) (err error)

+ func (m *ModelDbX) Fields() (fields []string)

+ func (m *ModelDbX) Get(db *sqlx.DB, fields ...string) (err error)

+ func (m *ModelDbX) Insert(db *sqlx.DB, fields ...string) (err error)

+ func (m *ModelDbX) Rec() map[string]interface{}

+ func (m *ModelDbX) Select(db *sqlx.DB, objPtr interface{}, fields ...string) (err error)

+ func (m *ModelDbX) TableName(objPtr interface{}) string

+ func (m *ModelDbX) Update(db *sqlx.DB, fields ...string) (err error)

+ func (m *ModelDbX) Values(fields []string) (values []interface{}, err error)

+ func (m *ModelDbX) Where(where ...interface{}) *ModelDbX

type MsSQLServerConn

+ func (conn *MsSQLServerConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

type MySQLConn

+ func (conn *MySQLConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

type OracleConn

+ func (conn *OracleConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

type PostgresConn

+ func (conn *PostgresConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

type RedshiftConn

+ func (conn *RedshiftConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

+ type Result struct

+ func (r Result) LastInsertId() (int64, error)

+ func (r Result) RowsAffected() (int64, error)

type SQLiteConn

+ func (conn *SQLiteConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

type SnowflakeConn

+ func (conn *SnowflakeConn) GenerateUpsertSQL(srcTable string, tgtTable string, pkFields []string) (sql string, err error)

+ type Transaction struct

+ Conn Connection

+ Context *g.Context

+ Tx *sqlx.Tx

+ func (t *Transaction) Commit() (err error)

+ func (t *Transaction) DisableTrigger(tableName, triggerName string) (err error)

+ func (t *Transaction) EnableTrigger(tableName, triggerName string) (err error)

+ func (t *Transaction) Exec(sql string, args ...interface{}) (result sql.Result, err error)

+ func (t *Transaction) ExecContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (t *Transaction) ExecMultiContext(ctx context.Context, q string, args ...interface{}) (result sql.Result, err error)

+ func (t *Transaction) InsertBatchStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (t *Transaction) InsertStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (t *Transaction) Prepare(query string) (stmt *sql.Stmt, err error)

+ func (t *Transaction) QueryContext(ctx context.Context, q string, args ...interface{}) (result *sqlx.Rows, err error)

+ func (t *Transaction) Rollback() (err error)

+ func (t *Transaction) Upsert(sourceTable, targetTable string, pkFields []string) (count uint64, err error)

+ func (t *Transaction) UpsertStream(tableFName string, ds *iop.Datastream, pk []string) (count uint64, err error)

+ type User struct

+ Age int

+ Name string

+ func NewUser() *User

+ type WhereClause []interface

+ func (wc WhereClause) Args() []interface{}

+ func (wc WhereClause) Clause() string

Mar 8, 2021

Changes in this version

+ var SampleSize = 900

+ var UseBulkExportFlowCSV = false

+ func CleanSQL(conn Connection, sql string) string

+ func ColumnNames(columns []iop.Column) (colNames []string)

+ func CommonColumns(colNames1 []string, colNames2 []string) (commCols []string)

+ func CopyFromAzure(conn Connection, tableFName, azPath string) (err error)

+ func CopyFromS3(conn Connection, tableFName, s3Path string) (err error)

+ func GetSlingEnv() map[string]string

+ func SQLColumns(colTypes []*sql.ColumnType, NativeTypeMap map[string]string) (columns []iop.Column)

+ func SplitTableFullName(tableName string) (string, string)

+ func TestPermissions(conn Connection, tableName string) (err error)

+ type BaseConn struct

+ Data iop.Dataset

+ Type dbio.Type

+ URL string

+ func (conn *BaseConn) BaseURL() string

+ func (conn *BaseConn) Begin(options ...*sql.TxOptions) (err error)

+ func (conn *BaseConn) BulkExportFlow(sqls ...string) (df *iop.Dataflow, err error)

+ func (conn *BaseConn) BulkExportFlowCSV(sqls ...string) (df *iop.Dataflow, err error)

+ func (conn *BaseConn) BulkExportStream(sql string) (ds *iop.Datastream, err error)

+ func (conn *BaseConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *BaseConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *BaseConn) CastColumnForSelect(srcCol iop.Column, tgtCol iop.Column) string

+ func (conn *BaseConn) CastColumnsForSelect(srcColumns []iop.Column, tgtColumns []iop.Column) []string

+ func (conn *BaseConn) Close() error

+ func (conn *BaseConn) Commit() (err error)

+ func (conn *BaseConn) CompareChecksums(tableName string, columns []iop.Column) (err error)

+ func (conn *BaseConn) Connect(timeOut ...int) (err error)

+ func (conn *BaseConn) Context() *g.Context

+ func (conn *BaseConn) Db() *sqlx.DB

+ func (conn *BaseConn) DropTable(tableNames ...string) (err error)

+ func (conn *BaseConn) DropView(viewNames ...string) (err error)

+ func (conn *BaseConn) Exec(sql string, args ...interface{}) (result sql.Result, err error)

+ func (conn *BaseConn) ExecContext(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error)

+ func (conn *BaseConn) GenerateDDL(tableFName string, data iop.Dataset) (string, error)

+ func (conn *BaseConn) GenerateInsertStatement(tableName string, fields []string, numRows int) string

+ func (conn *BaseConn) GenerateUpsertExpressions(srcTable string, tgtTable string, pkFields []string) (exprs map[string]string, err error)

+ func (conn *BaseConn) GetColumnStats(tableName string, fields ...string) (columns []iop.Column, err error)

+ func (conn *BaseConn) GetColumns(tableFName string, fields ...string) (columns []iop.Column, err error)

+ func (conn *BaseConn) GetColumnsFull(tableFName string) (iop.Dataset, error)

+ func (conn *BaseConn) GetCount(tableFName string) (uint64, error)

+ func (conn *BaseConn) GetDDL(tableFName string) (string, error)

+ func (conn *BaseConn) GetGormConn(config *gorm.Config) (*gorm.DB, error)

+ func (conn *BaseConn) GetIndexes(tableFName string) (iop.Dataset, error)

+ func (conn *BaseConn) GetObjects(schema string, objectType string) (iop.Dataset, error)

+ func (conn *BaseConn) GetPrimaryKeys(tableFName string) (iop.Dataset, error)

+ func (conn *BaseConn) GetProp(key string) string

+ func (conn *BaseConn) GetSQLColumns(sqls ...string) (columns []iop.Column, err error)

+ func (conn *BaseConn) GetSchemaObjects(schemaName string) (Schema, error)

+ func (conn *BaseConn) GetSchemas() (iop.Dataset, error)

+ func (conn *BaseConn) GetTables(schema string) (iop.Dataset, error)

+ func (conn *BaseConn) GetTemplateValue(path string) (value string)

+ func (conn *BaseConn) GetType() dbio.Type

+ func (conn *BaseConn) GetURL(newURL ...string) string

+ func (conn *BaseConn) GetViews(schema string) (iop.Dataset, error)

+ func (conn *BaseConn) Import(data iop.Dataset, tableName string) error

+ func (conn *BaseConn) Init() (err error)

+ func (conn *BaseConn) InsertBatchStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *BaseConn) InsertStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *BaseConn) Kill() error

+ func (conn *BaseConn) LoadTemplates() error

+ func (conn *BaseConn) MustExec(sql string, args ...interface{}) (result sql.Result)

+ func (conn *BaseConn) OptimizeTable(tableName string, newColStats []iop.Column) (err error)

+ func (conn *BaseConn) Prepare(ctx context.Context, query string) (stmt *sql.Stmt, err error)

+ func (conn *BaseConn) PropArr() []string

+ func (conn *BaseConn) Props() map[string]string

+ func (conn *BaseConn) Query(sql string, limit ...int) (data iop.Dataset, err error)

+ func (conn *BaseConn) QueryContext(ctx context.Context, sql string, limit ...int) (iop.Dataset, error)

+ func (conn *BaseConn) Quote(field string) string

+ func (conn *BaseConn) Rollback() (err error)

+ func (conn *BaseConn) RunAnalysis(analysisName string, values map[string]interface{}) (iop.Dataset, error)

+ func (conn *BaseConn) RunAnalysisField(analysisName string, tableFName string, fields ...string) (iop.Dataset, error)

+ func (conn *BaseConn) RunAnalysisTable(analysisName string, tableFNames ...string) (iop.Dataset, error)

+ func (conn *BaseConn) Schemata() Schemata

+ func (conn *BaseConn) Self() Connection

+ func (conn *BaseConn) SetProp(key string, val string)

+ func (conn *BaseConn) StreamRecords(sql string) (<-chan map[string]interface{}, error)

+ func (conn *BaseConn) StreamRows(sql string, limit ...int) (ds *iop.Datastream, err error)

+ func (conn *BaseConn) StreamRowsContext(ctx context.Context, sql string, limit ...int) (ds *iop.Datastream, err error)

+ func (conn *BaseConn) SwapTable(srcTable string, tgtTable string) (err error)

+ func (conn *BaseConn) TableExists(tableFName string) (exists bool, err error)

+ func (conn *BaseConn) Template() Template

+ func (conn *BaseConn) Tx() *sqlx.Tx

+ func (conn *BaseConn) Unquote(field string) string

+ func (conn *BaseConn) Upsert(srcTable string, tgtTable string, primKeys []string) (rowAffCnt int64, err error)

+ func (conn *BaseConn) ValidateColumnNames(tgtColNames []string, colNames []string, quote bool) (newColNames []string, err error)

+ type BigQueryConn struct

+ Client *bigquery.Client

+ DatasetID string

+ Datasets []string

+ Location string

+ ProjectID string

+ URL string

+ func (conn *BigQueryConn) BulkExportFlow(sqls ...string) (df *iop.Dataflow, err error)

+ func (conn *BigQueryConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *BigQueryConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *BigQueryConn) Close() error

+ func (conn *BigQueryConn) Connect(timeOut ...int) error

+ func (conn *BigQueryConn) CopyFromGCS(gcsURI string, tableFName string, dsColumns []iop.Column) error

+ func (conn *BigQueryConn) CopyToGCS(tableFName string, gcsURI string) error

+ func (conn *BigQueryConn) ExecContext(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error)

+ func (conn *BigQueryConn) Init() error

+ func (conn *BigQueryConn) InsertBatchStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *BigQueryConn) InsertStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *BigQueryConn) StreamRowsContext(ctx context.Context, sql string, limit ...int) (ds *iop.Datastream, err error)

+ func (conn *BigQueryConn) Unload(sqls ...string) (gsPath string, err error)

+ func (conn *BigQueryConn) Upsert(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error)

+ type Connection interface

+ BaseURL func() string

+ Begin func(options ...*sql.TxOptions) error

+ BulkExportFlow func(sqls ...string) (*iop.Dataflow, error)

+ BulkExportStream func(sql string) (*iop.Datastream, error)

+ BulkImportFlow func(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ BulkImportStream func(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ CastColumnForSelect func(srcColumn iop.Column, tgtColumn iop.Column) string

+ CastColumnsForSelect func(srcColumns []iop.Column, tgtColumns []iop.Column) []string

+ Close func() error

+ Commit func() error

+ CompareChecksums func(tableName string, columns []iop.Column) (err error)

+ Connect func(timeOut ...int) error

+ Context func() *g.Context

+ Db func() *sqlx.DB

+ DropTable func(...string) error

+ DropView func(...string) error

+ Exec func(sql string, args ...interface{}) (result sql.Result, err error)

+ ExecContext func(ctx context.Context, sql string, args ...interface{}) (result sql.Result, err error)

+ GenerateDDL func(tableFName string, data iop.Dataset) (string, error)

+ GenerateInsertStatement func(tableName string, fields []string, numRows int) string

+ GenerateUpsertExpressions func(srcTable string, tgtTable string, pkFields []string) (exprs map[string]string, err error)

+ GetColumnStats func(tableName string, fields ...string) (columns []iop.Column, err error)

+ GetColumns func(tableFName string, fields ...string) ([]iop.Column, error)

+ GetColumnsFull func(string) (iop.Dataset, error)

+ GetCount func(string) (uint64, error)

+ GetDDL func(string) (string, error)

+ GetGormConn func(config *gorm.Config) (*gorm.DB, error)

+ GetIndexes func(string) (iop.Dataset, error)

+ GetPrimaryKeys func(string) (iop.Dataset, error)

+ GetProp func(string) string

+ GetSQLColumns func(sqls ...string) (columns []iop.Column, err error)

+ GetSchemaObjects func(string) (Schema, error)

+ GetSchemas func() (iop.Dataset, error)

+ GetTables func(string) (iop.Dataset, error)

+ GetTemplateValue func(path string) (value string)

+ GetType func() dbio.Type

+ GetURL func(newURL ...string) string

+ GetViews func(string) (iop.Dataset, error)

+ Init func() error

+ InsertBatchStream func(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ InsertStream func(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ Kill func() error

+ LoadTemplates func() error

+ MustExec func(sql string, args ...interface{}) (result sql.Result)

+ OptimizeTable func(tableName string, columns []iop.Column) (err error)

+ Props func() map[string]string

+ PropsArr func() []string

+ Query func(sql string, limit ...int) (iop.Dataset, error)

+ QueryContext func(ctx context.Context, sql string, limit ...int) (iop.Dataset, error)

+ Quote func(string) string

+ RenameTable func(table string, newTable string) (err error)

+ Rollback func() error

+ RunAnalysis func(string, map[string]interface{}) (iop.Dataset, error)

+ RunAnalysisField func(string, string, ...string) (iop.Dataset, error)

+ RunAnalysisTable func(string, ...string) (iop.Dataset, error)

+ Schemata func() Schemata

+ Self func() Connection

+ SetProp func(string, string)

+ StreamRecords func(sql string) (<-chan map[string]interface{}, error)

+ StreamRows func(sql string, limit ...int) (*iop.Datastream, error)

+ StreamRowsContext func(ctx context.Context, sql string, limit ...int) (ds *iop.Datastream, err error)

+ SwapTable func(srcTable string, tgtTable string) (err error)

+ TableExists func(tableFName string) (exists bool, err error)

+ Template func() Template

+ Tx func() *sqlx.Tx

+ Unquote func(string) string

+ Upsert func(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error)

+ ValidateColumnNames func(tgtColName []string, colNames []string, quote bool) (newColNames []string, err error)

+ func NewConn(URL string, props ...string) (Connection, error)

+ func NewConnContext(ctx context.Context, URL string, props ...string) (Connection, error)

+ type MsSQLServerConn struct

+ URL string

+ func (conn *MsSQLServerConn) BcpExport() (err error)

+ func (conn *MsSQLServerConn) BcpImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *MsSQLServerConn) BcpImportStreamParrallel(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *MsSQLServerConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *MsSQLServerConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *MsSQLServerConn) CopyFromAzure(tableFName, azPath string) (count uint64, err error)

+ func (conn *MsSQLServerConn) CopyViaAzure(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *MsSQLServerConn) GetURL(newURL ...string) string

+ func (conn *MsSQLServerConn) Init() error

+ func (conn *MsSQLServerConn) Upsert(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error)

+ type MySQLConn struct

+ URL string

+ func (conn *MySQLConn) BulkExportStream(sql string) (ds *iop.Datastream, err error)

+ func (conn *MySQLConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *MySQLConn) GetURL(newURL ...string) string

+ func (conn *MySQLConn) Init() error

+ func (conn *MySQLConn) LoadDataInFile(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *MySQLConn) LoadDataOutFile(sql string) (stdOutReader io.Reader, err error)

+ func (conn *MySQLConn) Upsert(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error)

+ type OracleConn struct

+ URL string

+ func (conn *OracleConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *OracleConn) GenerateInsertStatement(tableName string, fields []string, numRows int) string

+ func (conn *OracleConn) Init() error

+ func (conn *OracleConn) SQLLoad(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *OracleConn) Upsert(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error)

+ type Pool struct

+ Dbs map[string]*sqlx.DB

+ Mux sync.Mutex

+ type PostgresConn struct

+ URL string

+ func (conn *PostgresConn) BulkExportStream(sql string) (ds *iop.Datastream, err error)

+ func (conn *PostgresConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *PostgresConn) CastColumnForSelect(srcCol iop.Column, tgtCol iop.Column) (selectStr string)

+ func (conn *PostgresConn) CopyToStdout(sql string) (stdOutReader io.Reader, err error)

+ func (conn *PostgresConn) Init() error

+ func (conn *PostgresConn) Upsert(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error)

+ type RedshiftConn struct

+ URL string

+ func (conn *RedshiftConn) BulkExportFlow(sqls ...string) (df *iop.Dataflow, err error)

+ func (conn *RedshiftConn) BulkExportStream(sql string) (ds *iop.Datastream, err error)

+ func (conn *RedshiftConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *RedshiftConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *RedshiftConn) CopyFromS3(tableFName, s3Path string) (count uint64, err error)

+ func (conn *RedshiftConn) Init() error

+ func (conn *RedshiftConn) Unload(sqls ...string) (s3Path string, err error)

+ func (conn *RedshiftConn) Upsert(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error)

+ type SQLiteConn struct

+ URL string

+ func (conn *SQLiteConn) Init() error

+ func (conn *SQLiteConn) Upsert(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error)

+ type Schema struct

+ Name string

+ Tables map[string]Table

+ func (schema *Schema) ToData() (data iop.Dataset)

+ type Schemata struct

+ Schemas map[string]Schema

+ Tables map[string]*Table

+ type SnowflakeConn struct

+ CopyMethod string

+ URL string

+ Warehouse string

+ func (conn *SnowflakeConn) BulkExportFlow(sqls ...string) (df *iop.Dataflow, err error)

+ func (conn *SnowflakeConn) BulkImportFlow(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *SnowflakeConn) BulkImportStream(tableFName string, ds *iop.Datastream) (count uint64, err error)

+ func (conn *SnowflakeConn) Connect(timeOut ...int) error

+ func (conn *SnowflakeConn) CopyFromAzure(tableFName, azPath string) (err error)

+ func (conn *SnowflakeConn) CopyFromS3(tableFName, s3Path string) (err error)

+ func (conn *SnowflakeConn) CopyToAzure(sqls ...string) (azPath string, err error)

+ func (conn *SnowflakeConn) CopyToS3(sqls ...string) (s3Path string, err error)

+ func (conn *SnowflakeConn) CopyViaAWS(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *SnowflakeConn) CopyViaAzure(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *SnowflakeConn) CopyViaStage(tableFName string, df *iop.Dataflow) (count uint64, err error)

+ func (conn *SnowflakeConn) GetSchemas() (iop.Dataset, error)

+ func (conn *SnowflakeConn) GetTables(schema string) (iop.Dataset, error)

+ func (conn *SnowflakeConn) Init() error

+ func (conn *SnowflakeConn) PutFile(fPath string, internalStagePath string) (err error)

+ func (conn *SnowflakeConn) Upsert(srcTable string, tgtTable string, pkFields []string) (rowAffCnt int64, err error)

+ type Table struct

+ Columns []iop.Column

+ ColumnsMap map[string]*iop.Column

+ FullName string

+ IsView bool

+ Name string

+ type Template struct

+ Analysis map[string]string

+ Core map[string]string

+ Function map[string]string

+ GeneralTypeMap map[string]string

+ Metadata map[string]string

+ NativeStatsMap map[string]bool

+ NativeTypeMap map[string]string

+ Variable map[string]string

+ func (template Template) ToData() (data iop.Dataset)