macrobench

Documentation

Index

• Variables
• func Compare(client storage.SQLClient, old, new string, types []string, ...) (map[string]StatisticalCompareResults, error)
• func Run(mabcfg Config) error
• func Search(client storage.SQLClient, sha string, types []string, planner PlannerVersion) (map[string]StatisticalSingleResult, error)
• func SearchForLastDaysQPSOnly(client storage.SQLClient, types []string, planner PlannerVersion, days int) (map[string][]ShortStatisticalSingleResult, error)
• type Config
  • func (mabcfg *Config) AddToCommand(cmd *cobra.Command)
• type PlannerVersion
• type Range
• type ShortStatisticalSingleResult
• type StatisticalCompareResults
• type StatisticalResult
• type StatisticalSingleResult
  • func SearchForLastDays(client storage.SQLClient, macroType string, planner PlannerVersion, days int) ([]StatisticalSingleResult, error)
• type StatisticalSummary
• type Type
  • func (mbtype *Type) Set(s string) error
  • func (mbtype Type) String() string
  • func (mbtype Type) ToUpper() Type
  • func (mbtype *Type) Type() string
• type VTGateQueryPlan
  • func GetVTGateSelectQueryPlansWithFilter(gitRef string, macroType Type, planner PlannerVersion, ...) ([]VTGateQueryPlan, error)
• type VTGateQueryPlanComparer
  • func CompareVTGateQueryPlans(left, right []VTGateQueryPlan) []VTGateQueryPlanComparer
• type VTGateQueryPlanMap
• type VTGateQueryPlanValue

Variables

var (
	LegacyPlannerVersions = []PlannerVersion{
		V3Planner,
		Gen4FallbackPlanner,
	}
)

Functions

func Compare

func Compare(client storage.SQLClient, old, new string, types []string, planner PlannerVersion) (map[string]StatisticalCompareResults, error)

func Run

func Run(mabcfg Config) error

Run executes a macro benchmark by using sysbench. Based on the given MacroBenchConfig, the function will parse the configuration to send down to sysbench (size of tables duration of benchmark, mysql targets, etc...). After the execution, the output of the last step (stepRun) is converted to a slice of MacroBenchmarkResult, which is then uploaded to MySQL using the mysql.ConfigDB in MacroBenchConfig.

We use two forks of sysbench, one for oltp workloads and the other for tpcc workload. We use these forks because they implement a custom method to print results in JSON.

Regular Sysbench: https://github.com/planetscale/sysbench Sysbench-TPCC: https://github.com/planetscale/sysbench-tpcc

func Search

func Search(client storage.SQLClient, sha string, types []string, planner PlannerVersion) (map[string]StatisticalSingleResult, error)

func SearchForLastDaysQPSOnly

func SearchForLastDaysQPSOnly(client storage.SQLClient, types []string, planner PlannerVersion, days int) (map[string][]ShortStatisticalSingleResult, error)

Types

type Config

type Config struct {
	// SysbenchExec defines the path to sysbench binary
	SysbenchExec string

	// WorkloadPath defines the path to the lua file used by sysbench.
	WorkloadPath string

	// DatabaseConfig points to the configuration used to create
	// a *psdb.Client. If no configuration, results and reports will
	// not be saved to a database, though the program won't fail.
	DatabaseConfig *psdb.Config

	// MetricsDatabaseConfig points to the required configuration to create
	// a *influxdb.Client. If no configuration is provided results will not be
	// saved to the database and the program will not fail.
	MetricsDatabaseConfig *influxdb.Config

	// M contains all metadata used to parameter sysbench execution.
	// This key value map stores the value of each CLI parameters.
	M map[string]string

	// SkipSteps is a list of strings (separated by a comma) that is used to skip some of
	// sysbench steps.
	SkipSteps string

	// Type will be used to differentiate macro benchmarks.
	Type Type

	// Source defines from where the macro benchmark is triggered.
	// This field is used to distinguish runs triggered by webhooks,
	// local, nightly build, and so on.
	Source string

	// GitRef refers to the commit SHA pointing to the version
	// of Vitess that we are currently macro benchmarking.
	GitRef string

	// VtgatePlannerVersion refers to the planner version that the vtgate is using
	// in Vitess that we are currently macro benchmarking.
	VtgatePlannerVersion string

	// WorkingDirectory defines from where sysbench commands will be executed.
	// This parameter
	WorkingDirectory string
	// contains filtered or unexported fields
}

Config defines a configuration used to execute macro benchmark. For instance, the Run method uses MacroBenchConfig.

func (*Config) AddToCommand

func (mabcfg *Config) AddToCommand(cmd *cobra.Command)

AddToCommand will add the different CLI flags used by MacroBenchConfig into the given *cobra.Command.

type PlannerVersion

type PlannerVersion string

const (
	ErrorNoSysBenchResult = "no sysbench results were found"

	V3Planner           PlannerVersion = "V3"
	Gen4FallbackPlanner PlannerVersion = "Gen4Fallback"
	Gen4Planner         PlannerVersion = "Gen4"
)

type Range

type Range struct {
	Infinite bool    `json:"infinite"`
	Unknown  bool    `json:"unknown"`
	Value    float64 `json:"value"`
}

type ShortStatisticalSingleResult

type ShortStatisticalSingleResult struct {
	TotalQPS StatisticalSummary `json:"total_qps"`
}

type StatisticalCompareResults

type StatisticalCompareResults struct {
	TotalQPS  StatisticalResult `json:"total_qps"`
	ReadsQPS  StatisticalResult `json:"reads_qps"`
	WritesQPS StatisticalResult `json:"writes_qps"`
	OtherQPS  StatisticalResult `json:"other_qps"`

	TPS     StatisticalResult `json:"tps"`
	Latency StatisticalResult `json:"latency"`
	Errors  StatisticalResult `json:"errors"`

	TotalComponentsCPUTime StatisticalResult            `json:"total_components_cpu_time"`
	ComponentsCPUTime      map[string]StatisticalResult `json:"components_cpu_time"`

	TotalComponentsMemStatsAllocBytes StatisticalResult            `json:"total_components_mem_stats_alloc_bytes"`
	ComponentsMemStatsAllocBytes      map[string]StatisticalResult `json:"components_mem_stats_alloc_bytes"`
}

StatisticalCompareResults is the full representation of the results obtained by comparing two samples using the Mann Whitney U Test.

type StatisticalResult

type StatisticalResult struct {
	Insignificant bool               `json:"insignificant"`
	Delta         float64            `json:"delta"`
	P             float64            `json:"p"`
	N1            int                `json:"n1"`
	N2            int                `json:"n2"`
	Old           StatisticalSummary `json:"old"`
	New           StatisticalSummary `json:"new"`
}

type StatisticalSingleResult

type StatisticalSingleResult struct {
	GitRef string `json:"git_ref"`

	TotalQPS  StatisticalSummary `json:"total_qps"`
	ReadsQPS  StatisticalSummary `json:"reads_qps"`
	WritesQPS StatisticalSummary `json:"writes_qps"`
	OtherQPS  StatisticalSummary `json:"other_qps"`

	TPS     StatisticalSummary `json:"tps"`
	Latency StatisticalSummary `json:"latency"`
	Errors  StatisticalSummary `json:"errors"`

	TotalComponentsCPUTime StatisticalSummary            `json:"total_components_cpu_time"`
	ComponentsCPUTime      map[string]StatisticalSummary `json:"components_cpu_time"`

	TotalComponentsMemStatsAllocBytes StatisticalSummary            `json:"total_components_mem_stats_alloc_bytes"`
	ComponentsMemStatsAllocBytes      map[string]StatisticalSummary `json:"components_mem_stats_alloc_bytes"`
}

StatisticalSingleResult represents a single benchmark's statistical summary.

func SearchForLastDays

func SearchForLastDays(client storage.SQLClient, macroType string, planner PlannerVersion, days int) ([]StatisticalSingleResult, error)

type StatisticalSummary

type StatisticalSummary struct {
	Center     float64 `json:"center"`
	Confidence float64 `json:"confidence"`
	Range      Range   `json:"range"`
}

type Type

type Type string

Type determines the type of a macro benchmark.

func (*Type) Set

func (mbtype *Type) Set(s string) error

Set implements Cobra flag.Value interface.

func (Type) String

func (mbtype Type) String() string

String returns the given Type as a string. It also implements the flag.Value interface.

func (Type) ToUpper

func (mbtype Type) ToUpper() Type

ToUpper returns a new Type in upper case.

func (*Type) Type

func (mbtype *Type) Type() string

Type implements Cobra flag.Value interface.

type VTGateQueryPlan

type VTGateQueryPlan struct {
	Key   string
	Value VTGateQueryPlanValue
}

func GetVTGateSelectQueryPlansWithFilter

func GetVTGateSelectQueryPlansWithFilter(gitRef string, macroType Type, planner PlannerVersion, client storage.SQLClient) ([]VTGateQueryPlan, error)

type VTGateQueryPlanComparer

type VTGateQueryPlanComparer struct {
	Left, Right      *VTGateQueryPlan
	SamePlan         bool
	Key              string
	ExecCountDiff    int
	ExecTimeDiff     int
	RowsReturnedDiff int
	ErrorsDiff       int
}

func CompareVTGateQueryPlans

func CompareVTGateQueryPlans(left, right []VTGateQueryPlan) []VTGateQueryPlanComparer

type VTGateQueryPlanMap

type VTGateQueryPlanMap map[string]VTGateQueryPlanValue

type VTGateQueryPlanValue

type VTGateQueryPlanValue struct {
	QueryType    string
	Original     string
	Instructions interface{}

	ExecCount    int // Count of times this plan was executed
	ExecTime     int // Average execution time per query
	ShardQueries int // Total number of shard queries
	RowsReturned int // Total number of rows
	RowsAffected int // Total number of rows
	Errors       int // Total number of errors

	TablesUsed interface{}
}