src

package
v0.2.1 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 15, 2024 License: Apache-2.0 Imports: 39 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	AnonymizeHashBytes = 4
	AnonymizeHashFmt   = "h%x" // add prefix 'h'
)
View Source 
const (
	ReplayResultFileExt = ".result"
)

Variables

View Source 
var (
	AnonymizeMinLength       int
	AnonymizerreserveIdHashs map[string]string

	// Identifiers that should not be anonymized.
	ReserveIdentifiers = lo.Map([]string{
		"olap",
		"global",
		"internal",
		"__internal_schema",
		"information_schema",
	}, func(s string, _ int) string { return strings.ToLower(s) })
)
View Source 
var (
	InternalSqlComment      = "/*dorisdump*/"
	InternalSqlCommentBytes = []byte(InternalSqlComment)
)

Functions

func Anonymize 

func Anonymize(method string, s string) string

func AnonymizeSql 

func AnonymizeSql(method string, sqlId, sql string) string

func Choose 

func Choose(msg string, items []string) (string, error)

func Confirm 

func Confirm(msg string) bool

func CountAuditlogs added in v0.2.1 

func CountAuditlogs(
	ctx context.Context,
	db *sqlx.DB,
	dbname, table string,
	opts AuditLogScanOpts,
) (int, error)

func DecodeReplaySqls 

func DecodeReplaySqls(
	s *bufio.Scanner,
	dbs, users map[string]struct{},
	from, to int64,
	maxCount int,
) (map[string][]*ReplaySql, int64, error)

func DetectCharset 

func DetectCharset(r *bufio.Reader) (string, error)

func EncodeReplaySql 

func EncodeReplaySql(ts, client, user, db, queryId, stmt string) string

func ExpandHome 

func ExpandHome(path string) string

func ExtractQueriesFromAuditLogs 

func ExtractQueriesFromAuditLogs(
	auditlogPaths []string,
	encoding string,
	opts AuditLogScanOpts,
	parallel int,
) ([][]string, error)

ExtractQueriesFromAuditLog extracts the query from an audit log.

func GetDBAuditLogs added in v0.2.1 

func GetDBAuditLogs(
	ctx context.Context,
	db *sqlx.DB,
	dbname, table string,
	opts AuditLogScanOpts,
	parallel int,
) ([]string, error)

func GetEncoding 

func GetEncoding(name string) (encoding.Encoding, error)

func GetLocalIP 

func GetLocalIP() string

func NewDB 

func NewDB(host string, port int16, user, password, db string) (*sqlx.DB, error)

func ParallelGroup 

func ParallelGroup(parallel int) *errgroup.Group

func RandStr 

func RandStr(length int) string

func ReplaySqls 

func ReplaySqls(
	ctx context.Context,
	host string, port int16, user, password string,
	resultDir string, client2sqls map[string][]*ReplaySql, speed float32, maxHashRows int,
	minTs int64, parallel int,
) error

func SanitizeLike 

func SanitizeLike(s string) string

func ScpFromRemote 

func ScpFromRemote(ctx context.Context, privKey, remoteUrl, localPath string) error

ScpFromRemote copies a file from a remote server to the local machine using scp. 

privKey is the path to the private key to use for authentication.
remoteUrl is the address of file on the remote server, format user@host:port/path.
localPath is the path of the local file to copy to.

func SetupAnonymizer 

func SetupAnonymizer(idMinLength int, reserveIds ...string)

func ShowDatabases 

func ShowDatabases(ctx context.Context, conn *sqlx.DB, dbnamePrefix string) ([]string, error)

func ShowFronendsDisksDir 

func ShowFronendsDisksDir(ctx context.Context, conn *sqlx.DB, diskType string) (dir string, err error)

func SshLs 

func SshLs(ctx context.Context, privKey, remoteUrl string) ([]string, error)

func WriteFile 

func WriteFile(path string, content string) error

Types

type AuditLogScanOpts added in v0.2.1 

type AuditLogScanOpts struct {
	// filter
	DBs                []string
	QueryMinDurationMs int
	QueryStates        []string
	OnlySelect         bool
	From, To           string

	Unescape bool
	Strict   bool
}

type AuditLogScanner 

type AuditLogScanner interface {
	Init()
	ScanOne(oneLine []byte) error
	Result() (sqls []string)
	Close()
}

func NewAuditLogScanner 

func NewAuditLogScanner(opts AuditLogScanOpts) AuditLogScanner

type BytesEncoder 

type BytesEncoder interface {
	Encode(b []byte) ([]byte, error)
}

func NewBytesEncoder 

func NewBytesEncoder(srcEncoding encoding.Encoding) BytesEncoder

type ColumnStats 

type ColumnStats struct {
	Name        string `yaml:"name"`
	Count       int64  `yaml:"-"`
	Ndv         int64  `yaml:"ndv"`
	NullCount   int64  `yaml:"null_count"`
	DataSize    int64  `yaml:"data_size"`
	AvgSizeByte int64  `yaml:"avg_size_byte"`
	Min         string `yaml:"min"`
	Max         string `yaml:"max"`
}

type DBSchema 

type DBSchema struct {
	Name    string        `yaml:"db"`
	Schemas []*Schema     `yaml:"-"`
	Stats   []*TableStats `yaml:"tables,omitempty"`
}

type DummyEncoder 

type DummyEncoder struct {
}

func (*DummyEncoder) Encode 

func (e *DummyEncoder) Encode(b []byte) ([]byte, error)

type ReplayClient 

type ReplayClient struct {
	// contains filtered or unexported fields
}

func (*ReplayClient) Close 

func (c *ReplayClient) Close(closefile bool)

type ReplayResult 

type ReplayResult struct {
	QueryId string `json:"queryId"`

	ReturnRows     int    `json:"returnRows"`
	ReturnRowsHash string `json:"returnRowsHash,omitempty"`
	DurationMs     int64  `json:"durationMs"`
	Err            string `json:"err,omitempty"`
}

func (*ReplayResult) String 

func (re *ReplayResult) String() string

type ReplaySql 

type ReplaySql struct {
	ReplaySqlMeta

	Stmt string
}

type ReplaySqlMeta 

type ReplaySqlMeta struct {
	Ts_     string `json:"ts"`
	Ts      int64  `json:"-"`
	Client  string `json:"client"`
	User    string `json:"user"`
	Db      string `json:"db"`
	QueryId string `json:"queryId"`
}

ReplaySqlMeta will be prepend to every sql as a comment.

e.g. "/*dorisdump{"ts": "2024-09-20 00:00:00", "client": "127.0.0.1:32345", "user": "root", "db": "test", "queryId": "1"}*/ <the sql>"

func (*ReplaySqlMeta) Timestamp 

func (m *ReplaySqlMeta) Timestamp() (ms int64, err error)

type Schema 

type Schema struct {
	Name       string     `db:"TABLE_NAME"`
	Type       SchemaType `db:"TABLE_TYPE"`
	DB         string     `db:"TABLE_SCHEMA"`
	CreateStmt string     `db:"-"`
}

func ShowCreateTables 

func ShowCreateTables(ctx context.Context, conn *sqlx.DB, db string, dbTables ...string) (schemas []*Schema, err error)

func ShowTables 

func ShowTables(ctx context.Context, conn *sqlx.DB, dbname string, tablenamePrefix ...string) (tables []*Schema, err error)

func (*Schema) String 

func (s *Schema) String() string

type SchemaType 

type SchemaType string
var (
	SchemaTypeTable            SchemaType = "TABLE"
	SchemaTypeView             SchemaType = "VIEW"
	SchemaTypeMaterializedView SchemaType = "MATERIALIZED_VIEW"
)

func (SchemaType) Lower 

func (s SchemaType) Lower() string

type SimpleAuditLogScanner 

type SimpleAuditLogScanner struct {
	AuditLogScanOpts
	// contains filtered or unexported fields
}

func NewSimpleAuditLogScanner 

func NewSimpleAuditLogScanner(opts AuditLogScanOpts) *SimpleAuditLogScanner

func (*SimpleAuditLogScanner) Close 

func (s *SimpleAuditLogScanner) Close()

func (*SimpleAuditLogScanner) Init 

func (s *SimpleAuditLogScanner) Init()

func (*SimpleAuditLogScanner) Result 

func (s *SimpleAuditLogScanner) Result() []string

func (*SimpleAuditLogScanner) ScanOne 

func (s *SimpleAuditLogScanner) ScanOne(oneLog []byte) error

type TableStats 

type TableStats struct {
	Name     string         `yaml:"name"`
	RowCount int64          `yaml:"row_count"`
	Columns  []*ColumnStats `yaml:"columns,omitempty"`
}

func GetTablesStats 

func GetTablesStats(ctx context.Context, conn *sqlx.DB, dbname string, tables ...string) ([]*TableStats, error)

type Utf8Encoder 

type Utf8Encoder struct {
	// contains filtered or unexported fields
}

func (*Utf8Encoder) Encode 

func (e *Utf8Encoder) Encode(b []byte) ([]byte, error)

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.