ckissu

package
v0.0.0-...-8d187f3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 22, 2024 License: Apache-2.0 Imports: 12 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	INTERVAL_HOUR = 60
	INTERVAL_DAY  = 1440
	DEFAULT_TTL   = 168
	RETRY_COUNT   = 1

	MAX_JOB_COUNT   = 10
	MIN_ORG_PER_JOB = 8
)

Variables

View Source 
var AllColumnAdds = [][]*ColumnAdds{ColumnAdd64, ColumnAdd65, ColumnAdd66}
View Source 
var AllColumnDrops = [][]*ColumnDrop{getColumnDrops(nil)}
View Source 
var AllColumnMods = [][]*ColumnMod{}
View Source 
var AllColumnRenames = [][]*ColumnRename{getColumnRenames(ColumnRename65)}
View Source 
var AllDatasourceAdds = [][]*ColumnDatasourceAdd{getColumnDatasourceAdds(ColumnDatasourceAdd65)}
View Source 
var AllIndexAdds = [][]*IndexAdd{getIndexAdds(IndexAdd64), getIndexAdds(IndexAdd65)}
View Source 
var AllTableModTTLs = [][]*TableModTTL{}
View Source 
var AllTableRenames = []*TableRename{}
View Source 
var ColumnAdd64 = []*ColumnAdds{
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l7_flow_log", "l7_flow_log_local"},
		ColumnNames: []string{"syscall_coroutine_0", "syscall_coroutine_1", "trace_id_index"},
		ColumnType:  ckdb.UInt64,
	},
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l7_flow_log", "l7_flow_log_local"},
		ColumnNames: []string{"is_tls"},
		ColumnType:  ckdb.UInt8,
	},
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l4_flow_log", "l4_flow_log_local"},
		ColumnNames: []string{"rtt_client", "rtt_server"},
		ColumnType:  ckdb.Float64,
	},
	{
		Dbs:         []string{"profile"},
		Tables:      []string{"in_process", "in_process_local"},
		ColumnNames: []string{"gprocess_id"},
		ColumnType:  ckdb.UInt32,
	},
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l4_flow_log", "l4_flow_log_local"},
		ColumnNames: []string{"tls_rtt"},
		ColumnType:  ckdb.Float64,
	},
}
View Source 
var ColumnAdd65 = []*ColumnAdds{
	{
		Dbs:         []string{"flow_metrics"},
		Tables:      []string{"traffic_policy.1m", "traffic_policy.1m_local"},
		ColumnNames: []string{"tunnel_ip_id"},
		ColumnType:  ckdb.UInt16,
	},
	{
		Dbs:         []string{"event"},
		Tables:      []string{"perf_event", "perf_event_local", "event", "event_local"},
		ColumnNames: []string{"process_kname"},
		ColumnType:  ckdb.String,
	},
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l4_flow_log", "l4_flow_log_local"},
		ColumnNames: []string{"request_domain"},
		ColumnType:  ckdb.String,
	},
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l7_flow_log", "l7_flow_log_local"},
		ColumnNames: []string{"events"},
		ColumnType:  ckdb.String,
	},
	{
		Dbs: []string{"flow_metrics"},
		Tables: []string{"application.1m", "application.1m_local", "application_map.1m", "application_map.1m_local",
			"application.1s", "application.1s_local", "application_map.1s", "application_map.1s_local"},
		ColumnNames: []string{"biz_type"},
		ColumnType:  ckdb.UInt8,
	},
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l7_flow_log_local", "l7_flow_log"},
		ColumnNames: []string{"biz_type"},
		ColumnType:  ckdb.UInt8,
	},
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l7_flow_log_local", "l7_flow_log"},
		ColumnNames: []string{"captured_request_byte", "captured_response_byte"},
		ColumnType:  ckdb.UInt32,
	},
	{
		Dbs:         []string{"event"},
		Tables:      []string{"perf_event", "perf_event_local", "event", "event_local"},
		ColumnNames: []string{"_id"},
		ColumnType:  ckdb.UInt64,
	},

	{
		Dbs: []string{"flow_metrics"},
		Tables: []string{
			"application.1m", "application.1m_local", "application_map.1m", "application_map.1m_local",
			"application.1s", "application.1s_local", "application_map.1s", "application_map.1s_local",
			"traffic_policy.1m", "traffic_policy.1m_local",
			"network.1m", "network.1m_local", "network_map.1m", "network_map.1m_local",
			"network.1s", "network.1s_local", "network_map.1s", "network_map.1s_local",
		},
		ColumnNames:  []string{"team_id"},
		ColumnType:   ckdb.UInt16,
		DefaultValue: "1",
	},
	{
		Dbs:          []string{"flow_log"},
		Tables:       []string{"l4_flow_log", "l4_flow_log_local", "l7_flow_log_local", "l7_flow_log", "l4_packet_local", "l4_packet", "l7_packet_local", "l7_packet"},
		ColumnNames:  []string{"team_id"},
		ColumnType:   ckdb.UInt16,
		DefaultValue: "1",
	},
	{
		Dbs:          []string{"event"},
		Tables:       []string{"event_local", "event", "perf_event", "perf_event_local"},
		ColumnNames:  []string{"team_id"},
		ColumnType:   ckdb.UInt16,
		DefaultValue: "1",
	},
	{
		Dbs:          []string{"ext_metrics"},
		Tables:       []string{"metrics_local", "metrics"},
		ColumnNames:  []string{"team_id"},
		ColumnType:   ckdb.UInt16,
		DefaultValue: "1",
	},
	{
		Dbs:          []string{"profile"},
		Tables:       []string{"in_process_local", "in_process"},
		ColumnNames:  []string{"team_id"},
		ColumnType:   ckdb.UInt16,
		DefaultValue: "1",
	},
	{
		Dbs:          []string{"prometheus"},
		Tables:       []string{"samples_local", "samples"},
		ColumnNames:  []string{"team_id"},
		ColumnType:   ckdb.UInt16,
		DefaultValue: "1",
	},
	{
		Dbs: []string{"flow_tag"},
		Tables: []string{
			"event_custom_field_local", "event_custom_field",
			"event_custom_field_value_local", "event_custom_field_value",
			"ext_metrics_custom_field_local", "ext_metrics_custom_field",
			"ext_metrics_custom_field_value_local", "ext_metrics_custom_field_value",
			"flow_log_custom_field_local", "flow_log_custom_field",
			"flow_log_custom_field_value_local", "flow_log_custom_field_value",
			"prometheus_custom_field_local", "prometheus_custom_field",
			"prometheus_custom_field_value_local", "prometheus_custom_field_value",
			"profile_custom_field_local", "profile_custom_field",
			"profile_custom_field_value_local", "profile_custom_field_value",
		},
		ColumnNames:  []string{"team_id"},
		ColumnType:   ckdb.UInt16,
		DefaultValue: "1",
	},
}

when adding a new column, you need to check whether `ColumnDatasourceAdd65` also needs to be added.

View Source 
var ColumnAdd66 = []*ColumnAdds{
	{
		Dbs:         []string{"profile"},
		Tables:      []string{"in_process", "in_process_local"},
		ColumnNames: []string{"auto_instance_id", "auto_service_id"},
		ColumnType:  ckdb.UInt32,
	},
	{
		Dbs:         []string{"profile"},
		Tables:      []string{"in_process", "in_process_local"},
		ColumnNames: []string{"auto_instance_type", "auto_service_type"},
		ColumnType:  ckdb.UInt8,
	},
}
View Source 
var ColumnDatasourceAdd65 = []*ColumnDatasourceAdds{
	{
		ColumnNames:    []string{"agent_id", "capture_network_type_id"},
		OldColumnNames: []string{"vtap_id", "tap_type"},
		ColumnTypes:    []ckdb.ColumnType{ckdb.UInt16, ckdb.UInt8},
		OnlyMapTable:   false,
		OnlyAppTable:   false,
	},
	{
		ColumnNames:    []string{"observation_point", "capture_nic", "capture_nic_type"},
		OldColumnNames: []string{"tap_side", "tap_port", "tap_port_type"},
		ColumnTypes:    []ckdb.ColumnType{ckdb.LowCardinalityString, ckdb.UInt32, ckdb.UInt8},
		OnlyMapTable:   true,
		OnlyAppTable:   false,
	},
	{
		ColumnNames:    []string{"biz_type"},
		OldColumnNames: []string{""},
		ColumnTypes:    []ckdb.ColumnType{ckdb.UInt8},
		OnlyMapTable:   false,
		OnlyAppTable:   true,
	},
	{
		ColumnNames:    []string{"team_id"},
		OldColumnNames: []string{""},
		ColumnTypes:    []ckdb.ColumnType{ckdb.UInt16},
		OnlyMapTable:   false,
		OnlyAppTable:   false,
		DefaultValue:   "1",
	},
	{
		ColumnNames:      []string{"server_syn_miss", "client_ack_miss"},
		OldColumnNames:   []string{"", ""},
		ColumnTypes:      []ckdb.ColumnType{ckdb.UInt64, ckdb.UInt64},
		OnlyMapTable:     false,
		OnlyAppTable:     false,
		OnlyNetworkTable: true,
		IsMetrics:        true,
		IsSummable:       true,
	},
}
View Source 
var ColumnRename65 = []*ColumnRenames{
	{
		Db:             "flow_log",
		Tables:         []string{"l4_packet", "l4_packet_local", "l7_packet", "l7_packet_local"},
		OldColumnNames: []string{"vtap_id"},
		NewColumnNames: []string{"agent_id"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt16},
	},
	{
		Db:             "ext_metrics",
		Tables:         []string{"metrics", "metrics_local"},
		OldColumnNames: []string{"vtap_id"},
		NewColumnNames: []string{"agent_id"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt16},
	},
	{
		Db:             "prometheus",
		Tables:         []string{"samples", "samples_local"},
		OldColumnNames: []string{"vtap_id"},
		NewColumnNames: []string{"agent_id"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt16},
	},
	{
		Db:             "event",
		Tables:         []string{"event", "event_local", "perf_event", "perf_event_local"},
		OldColumnNames: []string{"vtap_id"},
		NewColumnNames: []string{"agent_id"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt16},
	},
	{
		Db:             "profile",
		Tables:         []string{"in_process", "in_process_local"},
		OldColumnNames: []string{"vtap_id"},
		NewColumnNames: []string{"agent_id"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt16},
	},

	{
		Db:             "flow_log",
		Tables:         []string{"l4_flow_log", "l4_flow_log_local"},
		OldColumnNames: []string{"tap_type"},
		NewColumnNames: []string{"capture_network_type_id"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt8},
	},
	{
		Db:             "flow_log",
		Tables:         []string{"l7_flow_log", "l7_flow_log_local"},
		OldColumnNames: []string{"tap_type"},
		NewColumnNames: []string{"capture_network_type_id"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt8},
	},
	{
		Db: "flow_metrics",
		Tables: []string{
			"application.1m_local", "application.1s_local", "network.1m_local", "network.1s_local",
			"application.1m", "application.1s", "network.1m", "network.1s",
		},
		OldColumnNames: []string{"vtap_id", "tap_type"},
		NewColumnNames: []string{"agent_id", "capture_network_type_id"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt16, ckdb.UInt8},
	},
	{
		Db:             "flow_metrics",
		Tables:         []string{"traffic_policy.1m_local", "traffic_policy.1m"},
		OldColumnNames: []string{"vtap_id"},
		NewColumnNames: []string{"agent_id"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt16},
	},

	{
		Db:             "flow_log",
		Tables:         []string{"l4_flow_log", "l4_flow_log_local", "l7_flow_log", "l7_flow_log_local"},
		OldColumnNames: []string{"vtap_id", "tap_side", "tap_port", "tap_port_type"},
		NewColumnNames: []string{"agent_id", "observation_point", "capture_nic", "capture_nic_type"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt16, ckdb.LowCardinalityString, ckdb.UInt32, ckdb.UInt8},
	},
	{
		Db: "flow_metrics",
		Tables: []string{
			"application_map.1m_local", "application_map.1s_local", "network_map.1m_local", "network_map.1s_local",
			"application_map.1m", "application_map.1s", "network_map.1m", "network_map.1s",
		},
		OldColumnNames: []string{"vtap_id", "tap_type", "tap_side", "tap_port", "tap_port_type"},
		NewColumnNames: []string{"agent_id", "capture_network_type_id", "observation_point", "capture_nic", "capture_nic_type"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt16, ckdb.UInt8, ckdb.LowCardinalityString, ckdb.UInt32, ckdb.UInt8},
	},
	{
		Db: "flow_metrics",
		Tables: []string{
			"network.1m_local", "network.1s_local", "network.1m", "network.1s",
			"network_map.1m_local", "network_map.1s_local", "network_map.1m", "network_map.1s",
		},
		OldColumnNames: []string{"client_syn_repeat", "server_syn_ack_repeat"},
		NewColumnNames: []string{"server_syn_miss", "client_ack_miss"},
		OldColumnTypes: []ckdb.ColumnType{ckdb.UInt64, ckdb.UInt64},
	},
}

the rename process: first creates a new column, and then copies the old column data to the new column.

View Source 
var IndexAdd64 = []*IndexAdds{
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l7_flow_log_local"},
		ColumnNames: []string{"trace_id", "x_request_id_0", "x_request_id_1", "request_resource", "request_domain", "endpoint"},
		IndexType:   ckdb.IndexBloomfilter,
	},
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l7_flow_log_local"},
		ColumnNames: []string{"_id", "trace_id_index"},
		IndexType:   ckdb.IndexMinmax,
	},
}
View Source 
var IndexAdd65 = []*IndexAdds{
	{
		Dbs:         []string{"flow_log"},
		Tables:      []string{"l4_flow_log_local"},
		ColumnNames: []string{"request_domain"},
		IndexType:   ckdb.IndexBloomfilter,
	},
}
View Source 
var TableRenames65 = []*TableRename{
	{
		OldDb: "flow_metrics",
		OldTables: []string{
			"vtap_app_port.1m_local", "vtap_app_port.1s_local", "vtap_app_edge_port.1m_local", "vtap_app_edge_port.1s_local",
			"vtap_flow_port.1m_local", "vtap_flow_port.1s_local", "vtap_flow_edge_port.1m_local", "vtap_flow_edge_port.1s_local",
			"vtap_acl.1m_local"},
		NewDb: "flow_metrics",
		NewTables: []string{
			"application.1m_local", "application.1s_local", "application_map.1m_local", "application_map.1s_local",
			"network.1m_local", "network.1s_local", "network_map.1m_local", "network_map.1s_local",
			"traffic_policy.1m_local"},
	},
}

Functions

func Exec 

func Exec(connect *sql.DB, sql string) (sql.Result, error)

func Query 

func Query(connect *sql.DB, sql string) (*sql.Rows, error)

Types

type ColumnAdd 

type ColumnAdd struct {
	Db           string
	Table        string
	ColumnName   string
	ColumnType   ckdb.ColumnType
	DefaultValue string
	IsMetrics    bool
	AggrFunc     string
}

type ColumnAdds 

type ColumnAdds struct {
	Dbs          []string
	Tables       []string
	ColumnNames  []string
	ColumnType   ckdb.ColumnType
	DefaultValue string
}

type ColumnDatasourceAdd 

type ColumnDatasourceAdd struct {
	ColumnName                                   string
	OldColumnName                                string
	ColumnType                                   ckdb.ColumnType
	OnlyMapTable, OnlyAppTable, OnlyNetworkTable bool
	DefaultValue                                 string
	IsMetrics                                    bool
	IsSummable                                   bool
}

type ColumnDatasourceAdds 

type ColumnDatasourceAdds struct {
	ColumnNames                                  []string
	OldColumnNames                               []string
	ColumnTypes                                  []ckdb.ColumnType
	OnlyMapTable, OnlyAppTable, OnlyNetworkTable bool
	DefaultValue                                 string
	IsMetrics                                    bool
	IsSummable                                   bool
}

type ColumnDrop 

type ColumnDrop struct {
	Db         string
	Table      string
	ColumnName string
}

type ColumnDrops 

type ColumnDrops struct {
	Dbs         []string
	Tables      []string
	ColumnNames []string
}

type ColumnMod 

type ColumnMod struct {
	Db            string
	Table         string
	ColumnName    string
	NewColumnType ckdb.ColumnType
	DropIndex     bool
}

type ColumnRename 

type ColumnRename struct {
	Db              string
	Table           string
	OldColumnName   string
	CheckColumnType bool
	OldColumnType   ckdb.ColumnType
	NewColumnName   string
	DropIndex       bool
	DropMvTable     bool
}

type ColumnRenames 

type ColumnRenames struct {
	Db              string
	Tables          []string
	OldColumnNames  []string
	CheckColumnType bool
	OldColumnTypes  []ckdb.ColumnType
	NewColumnNames  []string
	DropIndex       bool
	DropMvTable     bool
}

type DatasourceInfo 

type DatasourceInfo struct {
	// contains filtered or unexported fields
}

type IndexAdd 

type IndexAdd struct {
	Db         string
	Table      string
	ColumnName string
	IndexType  ckdb.IndexType
}

type IndexAdds 

type IndexAdds struct {
	Dbs         []string
	Tables      []string
	ColumnNames []string
	IndexType   ckdb.IndexType
}

type Issu 

type Issu struct {
	Connections common.DBs
	VersionMaps []map[string]string
	Addrs       []string
	// contains filtered or unexported fields
}

func NewCKIssu 

func NewCKIssu(cfg *config.Config) (*Issu, error)

func (*Issu) Close 

func (i *Issu) Close() error

func (*Issu) RunRenameTable 

func (i *Issu) RunRenameTable(ds *datasource.DatasourceManager) error

called in server/ingester/ingester/ingester.go, executed before Start()

func (*Issu) Start 

func (i *Issu) Start() error

type TableModTTL 

type TableModTTL struct {
	Db     string
	Table  string
	NewTTL int
}

type TableRename 

type TableRename struct {
	OldDb     string
	OldTables []string
	NewDb     string
	NewTables []string
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.