table

package
v0.7.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Feb 18, 2023 License: Apache-2.0 Imports: 12 Imported by: 0

Documentation

Index

Constants

View Source 
const AccountAll = ETLParamAccountAll
View Source 
const CsvExtension = ".csv"
View Source 
const ETLParamAccountAll = "*"
View Source 
const ETLParamTypeAll = MergeLogTypeALL
View Source 
const ExternalFilePath = "__mo_filepath"
View Source 
const FilenameElems = 3
View Source 
const FilenameElemsV2 = 4
View Source 
const FilenameIdxType = 2
View Source 
const FilenameSeparator = "_"
View Source 
const PathElems = 7
View Source 
const PathIdxAccount = 0
View Source 
const PathIdxFilename = 6
View Source 
const PathIdxTable = 5
View Source 
const TaeExtension = ".tae"

Variables

View Source 
var CommonCsvOptions = &CsvOptions{
	FieldTerminator: ',',
	EncloseRune:     '"',
	Terminator:      '\n',
}
View Source 
var ETLParamTSAll = time.Time{}
View Source 
var ExternalTableEngine = "EXTERNAL"
View Source 
var NSecString = func() string {
	timeMu.Lock()
	nsec := time.Now().Nanosecond()
	timeMu.Unlock()
	return fmt.Sprintf("%09d", nsec)
}
View Source 
var NormalTableEngine = "TABLE"

Functions

func GetExtension 

func GetExtension(ext string) string

func GetOptionFactory 

func GetOptionFactory(ctx context.Context, engine string) func(db string, tbl string, account string) TableOptions

func SetPathBuilder 

func SetPathBuilder(ctx context.Context, pathBuilder string) error

func String2Bytes 

func String2Bytes(s string) (ret []byte)

func Time2DatetimeString 

func Time2DatetimeString(t time.Time) string

Types

type AccountDatePathBuilder 

type AccountDatePathBuilder struct {
	PathBuilderConfig
}

func NewAccountDatePathBuilder 

func NewAccountDatePathBuilder(opts ...PathBuilderOption) *AccountDatePathBuilder

func (*AccountDatePathBuilder) Build 

func (b *AccountDatePathBuilder) Build(account string, typ MergeLogType, ts time.Time, db string, tblName string) string

func (*AccountDatePathBuilder) BuildETLPath 

func (b *AccountDatePathBuilder) BuildETLPath(db, name, account string) string

BuildETLPath implement PathBuilder

# account | typ | ts | table | filename like: * /* /*/*/* /metric /*.csv

func (*AccountDatePathBuilder) GetName 

func (b *AccountDatePathBuilder) GetName() string

func (*AccountDatePathBuilder) NewLogFilename 

func (b *AccountDatePathBuilder) NewLogFilename(name, nodeUUID, nodeType string, ts time.Time, extension string) string

func (*AccountDatePathBuilder) NewMergeFilename 

func (b *AccountDatePathBuilder) NewMergeFilename(timestampStart, timestampEnd, extension string) string

func (*AccountDatePathBuilder) ParsePath 

func (b *AccountDatePathBuilder) ParsePath(ctx context.Context, path string) (Path, error)

func (*AccountDatePathBuilder) SupportAccountStrategy 

func (b *AccountDatePathBuilder) SupportAccountStrategy() bool

func (*AccountDatePathBuilder) SupportMergeSplit 

func (b *AccountDatePathBuilder) SupportMergeSplit() bool

type ColType 

type ColType int
const (
	TDatetime ColType = iota
	TUint64
	TInt64
	TFloat64
	TJson
	TText
	TVarchar
)

func (*ColType) String 

func (c *ColType) String(precision int) string

func (*ColType) ToType 

func (c *ColType) ToType() types.Type

type Column 

type Column struct {
	Name    string
	ColType ColType
	// Precision default 0, usually for varchar
	Precision int
	Default   string
	Comment   string
	Alias     string // only use in view
}

func DatetimeColumn 

func DatetimeColumn(name, comment string) Column

func Int64Column 

func Int64Column(name, comment string) Column

func JsonColumn 

func JsonColumn(name, comment string) Column

func SpanIDStringColumn 

func SpanIDStringColumn(name, comment string) Column

func StringColumn 

func StringColumn(name, comment string) Column

func StringDefaultColumn 

func StringDefaultColumn(name, defaultVal, comment string) Column

func StringWithPrecision 

func StringWithPrecision(name string, precision int, comment string) Column

func TextColumn 

func TextColumn(name, comment string) Column

func UInt64Column 

func UInt64Column(name, comment string) Column

func UuidStringColumn 

func UuidStringColumn(name, comment string) Column

func ValueColumn 

func ValueColumn(name, comment string) Column

func (*Column) ToCreateSql 

func (col *Column) ToCreateSql(ctx context.Context) string

ToCreateSql return column scheme in create sql

  • case 1: `column_name` varchar(36) DEFAULT "def_val" COMMENT "what am I, with default."
  • case 2: `column_name` varchar(36) NOT NULL COMMENT "what am I. Without default, SO NOT NULL."

type CsvOptions 

type CsvOptions struct {
	FieldTerminator rune // like: ','
	EncloseRune     rune // like: '"'
	Terminator      rune // like: '\n'
}

type CsvTableOptions 

type CsvTableOptions struct {
	Formatter string
	DbName    string
	TblName   string
	Account   string
}

func (*CsvTableOptions) FormatDdl 

func (o *CsvTableOptions) FormatDdl(ddl string) string

func (*CsvTableOptions) GetCreateOptions 

func (o *CsvTableOptions) GetCreateOptions() string

func (*CsvTableOptions) GetTableOptions 

func (o *CsvTableOptions) GetTableOptions(builder PathBuilder) string

type DBTablePathBuilder 

type DBTablePathBuilder struct{}

func NewDBTablePathBuilder 

func NewDBTablePathBuilder() *DBTablePathBuilder

func (*DBTablePathBuilder) Build 

func (m *DBTablePathBuilder) Build(account string, typ MergeLogType, ts time.Time, db string, name string) string

func (*DBTablePathBuilder) BuildETLPath 

func (m *DBTablePathBuilder) BuildETLPath(db, name, account string) string

BuildETLPath implement PathBuilder

like: system/metric_*.csv

func (*DBTablePathBuilder) GetName 

func (m *DBTablePathBuilder) GetName() string

func (*DBTablePathBuilder) NewLogFilename 

func (m *DBTablePathBuilder) NewLogFilename(name, nodeUUID, nodeType string, ts time.Time, extension string) string

func (*DBTablePathBuilder) NewMergeFilename 

func (m *DBTablePathBuilder) NewMergeFilename(timestampStart, timestampEnd, extension string) string

func (*DBTablePathBuilder) ParsePath 

func (m *DBTablePathBuilder) ParsePath(ctx context.Context, path string) (Path, error)

func (*DBTablePathBuilder) SupportAccountStrategy 

func (m *DBTablePathBuilder) SupportAccountStrategy() bool

func (*DBTablePathBuilder) SupportMergeSplit 

func (m *DBTablePathBuilder) SupportMergeSplit() bool

type ETLPath 

type ETLPath struct {
	// contains filtered or unexported fields
}

func NewETLPath 

func NewETLPath(path string) *ETLPath

NewETLPath

path like: sys/[log|merged]/yyyy/mm/dd/table/***.csv ## idx: 0 1 2 3 4 5 6 filename like: {timestamp}_{node_uuid}_{node_type}.csv ## or: {timestamp_start}_{timestamp_end}_merged.csv

func (*ETLPath) Parse 

func (p *ETLPath) Parse(ctx context.Context) error

func (*ETLPath) Table 

func (p *ETLPath) Table() string

func (*ETLPath) Timestamp 

func (p *ETLPath) Timestamp() []string

type ExportRequests 

type ExportRequests []WriteRequest

type FilePathCfg 

type FilePathCfg struct {
	NodeUUID  string
	NodeType  string
	Extension string
}

func (*FilePathCfg) LogsFilePathFactory 

func (c *FilePathCfg) LogsFilePathFactory(account string, tbl *Table, ts time.Time) string

type MergeLogType 

type MergeLogType string
const MergeLogTypeALL MergeLogType = "*"
const MergeLogTypeLogs MergeLogType = "logs"
const MergeLogTypeMerged MergeLogType = "merged"

func (MergeLogType) String 

func (t MergeLogType) String() string

type NoopTableOptions 

type NoopTableOptions struct{}

func (NoopTableOptions) FormatDdl 

func (o NoopTableOptions) FormatDdl(ddl string) string

func (NoopTableOptions) GetCreateOptions 

func (o NoopTableOptions) GetCreateOptions() string

func (NoopTableOptions) GetTableOptions 

func (o NoopTableOptions) GetTableOptions(PathBuilder) string

type Path 

type Path interface {
	Table() string
	Timestamp() []string
}

type PathBuilder 

type PathBuilder interface {
	// Build directory path
	Build(account string, typ MergeLogType, ts time.Time, db string, name string) string
	// BuildETLPath return path for EXTERNAL table 'infile' options
	//
	// like: {account}/merged/*/*/*/{name}/*.csv
	BuildETLPath(db, name, account string) string
	// ParsePath
	//
	// switch path {
	// case "{timestamp_writedown}_{node_uuid}_{ndoe_type}.csv":
	// case "{timestamp_start}_{timestamp_end}_merged.csv"
	// }
	ParsePath(ctx context.Context, path string) (Path, error)
	NewMergeFilename(timestampStart, timestampEnd, extension string) string
	NewLogFilename(name, nodeUUID, nodeType string, ts time.Time, extension string) string
	// SupportMergeSplit const. if false, not support SCV merge|split task
	SupportMergeSplit() bool
	// SupportAccountStrategy const
	SupportAccountStrategy() bool
	// GetName const
	GetName() string
}

PathBuilder hold strategy to build filepath

func PathBuilderFactory 

func PathBuilderFactory(pathBuilder string) PathBuilder

type PathBuilderConfig 

type PathBuilderConfig struct {
	// contains filtered or unexported fields
}

type PathBuilderOption 

type PathBuilderOption func(*PathBuilderConfig)

func WithDatabase 

func WithDatabase(with bool) PathBuilderOption

func (PathBuilderOption) Apply 

func (opt PathBuilderOption) Apply(cfg *PathBuilderConfig)

type Row 

type Row struct {
	Table *Table

	Columns    []any
	CsvColumns []string
}

func NewRow 

func NewRow() *Row

func (*Row) Clone 

func (r *Row) Clone() *Row

func (*Row) CsvPrimaryKey 

func (r *Row) CsvPrimaryKey() string

func (*Row) Free 

func (r *Row) Free()

func (*Row) GetAccount 

func (r *Row) GetAccount() string

GetAccount return r.Columns[r.AccountIdx] if r.AccountIdx >= 0 and r.Table.PathBuilder.SupportAccountStrategy, else return "sys"

func (*Row) GetCsvStrings 

func (r *Row) GetCsvStrings() []string

GetCsvStrings not format

func (*Row) GetRawColumn 

func (r *Row) GetRawColumn() []any

func (*Row) ParseRow 

func (r *Row) ParseRow(cols []string) error

func (*Row) Reset 

func (r *Row) Reset()

func (*Row) SetColumnVal 

func (r *Row) SetColumnVal(col Column, val any)

func (*Row) SetVal 

func (r *Row) SetVal(col string, val any)

func (*Row) Size 

func (r *Row) Size() (size int64)

func (*Row) ToStrings 

func (r *Row) ToStrings() []string

ToStrings output all column as string

type RowField 

type RowField interface {
	GetTable() *Table
	FillRow(context.Context, *Row)
}

type RowRequest 

type RowRequest struct {
	// contains filtered or unexported fields
}

func NewRowRequest 

func NewRowRequest(writer RowWriter) *RowRequest

func (*RowRequest) GetContent 

func (r *RowRequest) GetContent() string

func (*RowRequest) Handle 

func (r *RowRequest) Handle() (int, error)

type RowWriter 

type RowWriter interface {
	WriteRow(row *Row) error
	// GetContent get buffer content
	GetContent() string
	// FlushAndClose flush its buffer and close.
	FlushAndClose() (int, error)
}

type Table 

type Table struct {
	Account          string
	Database         string
	Table            string
	Columns          []Column
	PrimaryKeyColumn []Column
	Engine           string
	Comment          string
	// PathBuilder help to desc param 'infile'
	PathBuilder PathBuilder
	// AccountColumn help to split data in account's filepath
	AccountColumn *Column
	// TableOptions default is nil, see GetTableOptions
	TableOptions TableOptions
	// SupportUserAccess default false. if true, user account can access.
	SupportUserAccess bool
	// SupportConstAccess default false. if true, use Table.Account
	SupportConstAccess bool
	// contains filtered or unexported fields
}

func GetAllTable 

func GetAllTable() []*Table

func GetTable 

func GetTable(b string) (*Table, bool)

func RegisterTableDefine 

func RegisterTableDefine(table *Table) *Table

RegisterTableDefine return old one, if already registered

func (*Table) Clone 

func (tbl *Table) Clone() *Table

func (*Table) GetDatabase 

func (tbl *Table) GetDatabase() string

func (*Table) GetIdentify 

func (tbl *Table) GetIdentify() string

GetIdentify return identify like database.table

func (*Table) GetName 

func (tbl *Table) GetName() string

func (*Table) GetRow 

func (tbl *Table) GetRow(ctx context.Context) *Row

func (*Table) GetTableOptions 

func (tbl *Table) GetTableOptions(ctx context.Context) TableOptions

func (*Table) ToCreateSql 

func (tbl *Table) ToCreateSql(ctx context.Context, ifNotExists bool) string

type TableOptions 

type TableOptions interface {
	FormatDdl(ddl string) string
	// GetCreateOptions return option for `create {option}table`, which should end with ' '
	GetCreateOptions() string
	GetTableOptions(PathBuilder) string
}

type View 

type View struct {
	Database    string
	Table       string
	OriginTable *Table
	Columns     []Column
	Condition   WhereCondition
	// SupportUserAccess default false. if true, user account can access.
	SupportUserAccess bool
}

func (*View) ToCreateSql 

func (tbl *View) ToCreateSql(ctx context.Context, ifNotExists bool) string

type ViewOption 

type ViewOption func(view *View)

func SupportUserAccess 

func SupportUserAccess(support bool) ViewOption

func WithColumn 

func WithColumn(c Column) ViewOption

func (ViewOption) Apply 

func (opt ViewOption) Apply(view *View)

type ViewSingleCondition 

type ViewSingleCondition struct {
	Column Column
	Table  string
}

func (*ViewSingleCondition) String 

func (tbl *ViewSingleCondition) String() string

type WhereCondition 

type WhereCondition interface {
	String() string
}

type WriteRequest 

type WriteRequest interface {
	Handle() (int, error)
	GetContent() string
}

type WriterFactory 

type WriterFactory func(ctx context.Context, account string, tbl *Table, ts time.Time) RowWriter

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.