dbio

README

dbio

dbio is a golang library for all things Database, Storage I/O processing.

The particularly useful struct used are:

• Datastream: https://github.com/slingdata-io/sling-cli/core/dbio/blob/main/iop/datastream.go
• Dataflow: https://github.com/slingdata-io/sling-cli/core/dbio/blob/main/iop/dataflow.go

Projects using dbio:

• https://slingdata.io
• https://github.com/dbnet-io/dbnet
• https://github.com/dbrest-io/dbREST

Documentation

Index

• Variables
• type FileType
  • func (ft FileType) Ext() string
  • func (ft FileType) IsJson() bool
• type Kind
• type Template
  • func (template Template) Value(path string) (value string)
• type Type
  • func ValidateType(tStr string) (Type, bool)
  • func (t Type) DBNameUpperCase() bool
  • func (t Type) DefPort() int
  • func (t Type) GetTemplateValue(path string) (value string)
  • func (t Type) IsDb() bool
  • func (t Type) IsFile() bool
  • func (t Type) IsNoSQL() bool
  • func (t Type) IsUnknown() bool
  • func (t Type) Kind() Kind
  • func (t Type) Name() string
  • func (t Type) NameLong() string
  • func (t Type) Quote(field string, normalize ...bool) string
  • func (t Type) QuoteNames(names ...string) (newNames []string)
  • func (t Type) String() string
  • func (t Type) Template() (template Template, err error)
  • func (t Type) Unquote(field string) string

Variables

var AllFileType = []struct {
	Value  FileType
	TSName string
}{
	{FileTypeNone, "FileTypeNone"},
	{FileTypeCsv, "FileTypeCsv"},
	{FileTypeXml, "FileTypeXml"},
	{FileTypeExcel, "FileTypeExcel"},
	{FileTypeJson, "FileTypeJson"},
	{FileTypeParquet, "FileTypeParquet"},
	{FileTypeAvro, "FileTypeAvro"},
	{FileTypeSAS, "FileTypeSAS"},
	{FileTypeJsonLines, "FileTypeJsonLines"},
	{FileTypeIceberg, "FileTypeIceberg"},
	{FileTypeDelta, "FileTypeDelta"},
	{FileTypeRaw, "FileTypeRaw"},
}

var AllKind = []struct {
	Value  Kind
	TSName string
}{
	{KindDatabase, "KindDatabase"},
	{KindFile, "KindFile"},
	{KindUnknown, "KindUnknown"},
}

var AllType = []struct {
	Value  Type
	TSName string
}{
	{TypeUnknown, "TypeUnknown"},
	{TypeFileLocal, "TypeFileLocal"},
	{TypeFileHDFS, "TypeFileHDFS"},
	{TypeFileS3, "TypeFileS3"},
	{TypeFileAzure, "TypeFileAzure"},
	{TypeFileGoogle, "TypeFileGoogle"},
	{TypeFileFtp, "TypeFileFtp"},
	{TypeFileSftp, "TypeFileSftp"},
	{TypeFileHTTP, "TypeFileHTTP"},
	{TypeDbPostgres, "TypeDbPostgres"},
	{TypeDbRedshift, "TypeDbRedshift"},
	{TypeDbStarRocks, "TypeDbStarRocks"},
	{TypeDbMySQL, "TypeDbMySQL"},
	{TypeDbMariaDB, "TypeDbMariaDB"},
	{TypeDbOracle, "TypeDbOracle"},
	{TypeDbBigTable, "TypeDbBigTable"},
	{TypeDbBigQuery, "TypeDbBigQuery"},
	{TypeDbSnowflake, "TypeDbSnowflake"},
	{TypeDbSQLite, "TypeDbSQLite"},
	{TypeDbD1, "TypeDbD1"},
	{TypeDbDuckDb, "TypeDbDuckDb"},
	{TypeDbMotherDuck, "TypeDbMotherDuck"},
	{TypeDbSQLServer, "TypeDbSQLServer"},
	{TypeDbAzure, "TypeDbAzure"},
	{TypeDbAzureDWH, "TypeDbAzureDWH"},
	{TypeDbTrino, "TypeDbTrino"},
	{TypeDbClickhouse, "TypeDbClickhouse"},
	{TypeDbElasticsearch, "TypeDbElasticsearch"},
	{TypeDbMongoDB, "TypeDbMongoDB"},
	{TypeDbPrometheus, "TypeDbPrometheus"},
	{TypeDbProton, "TypeDbProton"},
}

Types

type FileType

type FileType string

const (
	FileTypeNone      FileType = ""
	FileTypeCsv       FileType = "csv"
	FileTypeXml       FileType = "xml"
	FileTypeExcel     FileType = "xlsx"
	FileTypeJson      FileType = "json"
	FileTypeParquet   FileType = "parquet"
	FileTypeAvro      FileType = "avro"
	FileTypeSAS       FileType = "sas7bdat"
	FileTypeJsonLines FileType = "jsonlines"
	FileTypeIceberg   FileType = "iceberg"
	FileTypeDelta     FileType = "delta"
	FileTypeRaw       FileType = "raw"
)

func (FileType) Ext

func (ft FileType) Ext() string

func (FileType) IsJson

func (ft FileType) IsJson() bool

type Kind

type Kind string

Kind is the connection kind

const (
	// KindDatabase for databases
	KindDatabase Kind = "database"
	// KindFile for files (cloud, sftp)
	KindFile Kind = "file"
	// KindUnknown for unknown
	KindUnknown Kind = ""
)

type Template

type Template struct {
	Core           map[string]string `yaml:"core"`
	Metadata       map[string]string `yaml:"metadata"`
	Analysis       map[string]string `yaml:"analysis"`
	Function       map[string]string `yaml:"function"`
	GeneralTypeMap map[string]string `yaml:"general_type_map"`
	NativeTypeMap  map[string]string `yaml:"native_type_map"`
	NativeStatsMap map[string]bool   `yaml:"native_stat_map"`
	Variable       map[string]string `yaml:"variable"`
}

Template is a database YAML template

func (Template) Value

func (template Template) Value(path string) (value string)

ToData convert is dataset

type Type

type Type string

Type is the connection type

const (
	TypeUnknown Type = ""

	TypeFileLocal  Type = "file"
	TypeFileHDFS   Type = "hdfs"
	TypeFileS3     Type = "s3"
	TypeFileAzure  Type = "azure"
	TypeFileGoogle Type = "gs"
	TypeFileFtp    Type = "ftp"
	TypeFileSftp   Type = "sftp"
	TypeFileHTTP   Type = "http"

	TypeDbPostgres      Type = "postgres"
	TypeDbRedshift      Type = "redshift"
	TypeDbStarRocks     Type = "starrocks"
	TypeDbMySQL         Type = "mysql"
	TypeDbMariaDB       Type = "mariadb"
	TypeDbOracle        Type = "oracle"
	TypeDbBigTable      Type = "bigtable"
	TypeDbBigQuery      Type = "bigquery"
	TypeDbSnowflake     Type = "snowflake"
	TypeDbSQLite        Type = "sqlite"
	TypeDbD1            Type = "d1"
	TypeDbDuckDb        Type = "duckdb"
	TypeDbMotherDuck    Type = "motherduck"
	TypeDbSQLServer     Type = "sqlserver"
	TypeDbAzure         Type = "azuresql"
	TypeDbAzureDWH      Type = "azuredwh"
	TypeDbTrino         Type = "trino"
	TypeDbClickhouse    Type = "clickhouse"
	TypeDbMongoDB       Type = "mongodb"
	TypeDbElasticsearch Type = "elasticsearch"
	TypeDbPrometheus    Type = "prometheus"
	TypeDbProton        Type = "proton"
)

func ValidateType

func ValidateType(tStr string) (Type, bool)

ValidateType returns true is type is valid

func (Type) DBNameUpperCase

func (t Type) DBNameUpperCase() bool

DBNameUpperCase returns true is upper case is default

func (Type) DefPort

func (t Type) DefPort() int

DefPort returns the default port

func (Type) GetTemplateValue

func (t Type) GetTemplateValue(path string) (value string)

func (Type) IsDb

func (t Type) IsDb() bool

IsDb returns true if database connection

func (Type) IsFile

func (t Type) IsFile() bool

IsFile returns true if file connection

func (Type) IsNoSQL

func (t Type) IsNoSQL() bool

IsDb returns true if database connection

func (Type) IsUnknown

func (t Type) IsUnknown() bool

IsUnknown returns true if unknown

func (Type) Kind

func (t Type) Kind() Kind

Kind returns the kind of connection

func (Type) Name

func (t Type) Name() string

Name return the type name

func (Type) NameLong

func (t Type) NameLong() string

NameLong return the type long name

func (Type) Quote

func (t Type) Quote(field string, normalize ...bool) string

Quote adds quotes to the field name

func (Type) QuoteNames

func (t Type) QuoteNames(names ...string) (newNames []string)

func (Type) String

func (t Type) String() string

String returns string instance

func (Type) Template

func (t Type) Template() (template Template, err error)

func (Type) Unquote

func (t Type) Unquote(field string) string

Unquote removes quotes to the field name