README
¶
Mutation testing support
This package contains rudimentary support for mutation-testing compilers. At the moment, it's highly specialised to one particular test campaign using a run-time mutated LLVM, but any diversification is helpful.
Documentation
¶
Overview ¶
Package mutation contains support for mutation testing using c4t.
Index ¶
Examples ¶
Constants ¶
const ( // MutantHitPrefix is the prefix of lines from compilers specifying that a mutant has been hit. MutantHitPrefix = "MUTATION HIT:" // MutantSelectPrefix is the prefix of lines from compilers specifying that a mutant has been selected. MutantSelectPrefix = "MUTATION SELECTED:" )
const EnvVar = "C4_MUTATION"
EnvVar is the environment variable used for mutation testing.
Some day, this might not be hard-coded.
Variables ¶
var ErrNotActive = errors.New("automation is disabled in this config")
Functions ¶
func ScanLine ¶
ScanLine scans line for mutant hit and selection hints, and calls the appropriate callback.
func ScanLines ¶
ScanLines scans each line in r, building a map of mutant indices to hit counts. If a mutant is present in the map, it was selected, even if its hit count is 0.
Example ¶
ExampleScanLines is a testable example for ScanLines.
package main
import (
"fmt"
"strings"
"github.com/c4-project/c4t/internal/mutation"
)
func main() {
lines := []string{
"warning: overfull hbox",
"MUTATION SELECTED: 42",
"warning: ineffective assign",
"MUTATION HIT: 42 (barely)",
"info: don't do this",
"this statement is false",
"MUTATION SELECTED: 8",
"MUTATION HIT: 42 (somewhat)",
}
for mutant, hits := range mutation.ScanLines(strings.NewReader(strings.Join(lines, "\n"))) {
fmt.Println(mutant, "=", hits)
}
}
Output: 42 = 2 8 = 0
Types ¶
type Analysis ¶
type Analysis map[Index]MutantAnalysis
Analysis is the type of mutation testing analyses.
func (Analysis) AddCompilation ¶
AddCompilation merges any mutant information extracted from log to this analysis. Such analysis is filed under compilation name comp, and status determines the status of the compilation.
Example ¶
ExampleAnalysis_AddCompilation is a testable example for AddCompilation.
package main
import (
"fmt"
"strings"
"github.com/c4-project/c4t/internal/mutation"
"github.com/c4-project/c4t/internal/subject/status"
"github.com/c4-project/c4t/internal/id"
"github.com/c4-project/c4t/internal/subject/compilation"
)
func main() {
log := strings.Join([]string{
"warning: overfull hbox",
"MUTATION SELECTED: 42",
"warning: ineffective assign",
"MUTATION HIT: 42 (barely)",
"info: don't do this",
"this statement is false",
"MUTATION SELECTED: 8",
"MUTATION HIT: 42 (somewhat)",
}, "\n")
ana := mutation.Analysis{}
ana.RegisterMutant(mutation.NamedMutant(42, "XYZ", 0))
fmt.Println("kills after 0 adds:", ana.Kills())
ana.AddCompilation(compilation.Name{SubjectName: "foo", CompilerID: id.FromString("gcc")}, log, status.Ok)
fmt.Println("kills after 1 adds:", ana.Kills())
ana.AddCompilation(compilation.Name{SubjectName: "bar", CompilerID: id.FromString("clang")}, log, status.Flagged)
fmt.Println("kills after 2 adds:", ana.Kills())
for _, ma := range ana {
fmt.Printf("%s:", ma.Mutant)
for _, h := range ma.Selections {
fmt.Printf(" [%dx, %s, killed: %v]", h.NumHits, h.HitBy, h.Killed())
}
fmt.Println()
}
}
Output: kills after 0 adds: [] kills after 1 adds: [] kills after 2 adds: [XYZ:42] XYZ:42: [2x, foo@gcc, killed: false] [2x, bar@clang, killed: true] 8: [0x, foo@gcc, killed: false] [0x, bar@clang, killed: false]
func (Analysis) RegisterMutant ¶
RegisterMutant registers the mutant record m in the analysis.
This is necessary, at the moment, to put things like the mutant's operator and variant information in the analysis table.
type AutoConfig ¶
type AutoConfig struct {
// Ranges contains the list of mutation number ranges that the campaign should use for automatic mutant selection.
Ranges []Range `json:"ranges,omitempty" toml:"ranges,omitempty"`
// ChangeMutantAfter is the (minimum) duration that each mutant gets before being automatically incremented.
// If 0, this sort of auto-increment
ChangeAfter quantity.Timeout `json:"change_after,omitempty" toml:"change_after,omitempty"`
// ChangeKilled specifies whether mutants should be automatically incremented after being killed.
ChangeKilled bool `json:"change_killed" toml:"change_killed"`
}
AutoConfig specifies configuration pertaining to automatically selecting mutants.
The mutation tester can be used with a manual selection, but is probably not very exciting.
func (AutoConfig) HasChangeAfter ¶
func (c AutoConfig) HasChangeAfter() bool
HasChangeAfter gets whether ChangeAfter is set to something other than zero.
func (AutoConfig) HasRanges ¶
func (c AutoConfig) HasRanges() bool
HasRanges gets whether at least one viable range exists, without expanding the ranges themselves.
func (AutoConfig) IsActive ¶
func (c AutoConfig) IsActive() bool
IsActive gets whether automatic selection is enabled.
Example ¶
ExampleAutoConfig_IsActive is a runnable example for Config.IsActive.
package main
import (
"fmt"
"time"
"github.com/c4-project/c4t/internal/quantity"
"github.com/c4-project/c4t/internal/mutation"
)
func main() {
cfg := mutation.AutoConfig{
Ranges: []mutation.Range{{Start: 1, End: 4}},
}
fmt.Println("disabled with ranges:", cfg.IsActive())
cfg.ChangeKilled = true
fmt.Println("after-killed with ranges:", cfg.IsActive())
cfg.ChangeKilled = false
cfg.ChangeAfter = quantity.Timeout(1 * time.Minute)
fmt.Println("after-time with ranges:", cfg.IsActive())
cfg.Ranges[0].Start = 4
fmt.Println("after-time with bad ranges:", cfg.IsActive())
}
Output: disabled with ranges: false after-killed with ranges: true after-time with ranges: true after-time with bad ranges: false
func (AutoConfig) Mutants ¶
func (c AutoConfig) Mutants() []Mutant
Mutants returns a list of all mutant numbers to consider in this testing campaign.
Mutants appear in the order defined, without deduplication. If Enabled is false, Mutants will be empty.
Example ¶
ExampleAutoConfig_Mutants is a runnable example for Config.Mutants.
package main
import (
"fmt"
"github.com/c4-project/c4t/internal/mutation"
)
func main() {
cfg := mutation.AutoConfig{
Ranges: []mutation.Range{
{Start: 1, End: 2},
{Operator: "FOO", Start: 2, End: 3},
{Start: 10, End: 12},
{Operator: "BAR", Start: 27, End: 31},
},
}
fmt.Print("mutants:")
for _, i := range cfg.Mutants() {
fmt.Printf(" %s", i)
}
fmt.Println()
}
Output: mutants: 1 FOO:2 10 11 BAR1:27 BAR2:28 BAR3:29 BAR4:30
type AutoPool ¶
type AutoPool struct {
// contains filtered or unexported fields
}
AutoPool manages the next mutant to select in an automated mutation testing campaign.
The policy AutoPool implements is: 1) Start by considering every mutant in turn. 2) Whenever a mutant is killed or its timeslot ends, advance to the next mutant. 3) If we are out of mutants, start again with the list of all mutants not killed by the steps above, and repeat. 4) If we kill every mutant, start again with every mutant. (This behaviour may change eventually.)
func (*AutoPool) Advance ¶
func (a *AutoPool) Advance()
Advance advances to the next mutant without killing it.
type Automator ¶
type Automator struct {
// TickerF is a stubbable function used to create a ticker.
TickerF func(time.Duration) (<-chan time.Time, Ticker)
// contains filtered or unexported fields
}
Automator handles most of the legwork of automating mutant selection.
func NewAutomator ¶
func NewAutomator(cfg AutoConfig) (*Automator, error)
NewAutomator constructs a new Automator given configuration cfg.
func (*Automator) KillCh ¶
KillCh gets a send channel for sending kill signals to this automator. If the automator isn't receiving kill signals, this will be nil. This channel must be closed.
type Config ¶
type Config struct {
// Enabled gets whether mutation testing is enabled.
//
// Setting this to false is equivalent to setting Ranges to empty.
Enabled bool `json:"enabled,omitempty" toml:"enabled,omitempty"`
// Selection contains any selected mutation.
//
// This can be set in the tester's config file, in addition to or instead of Ranges, but will be overridden by
// any automatic mutant selection.
Selection Mutant `json:"selection,omitempty" toml:"selection,omitempty"`
// Auto gathers configuration about how to automate mutation selection.
Auto AutoConfig `json:"auto,omitempty" toml:"auto,omitempty"`
}
Config configures a particular mutation testing campaign.
This currently just tracks ranges of mutation numbers, but may be generalised if we branch to supporting more than one kind of mutation test.
type Mutant ¶
type Mutant struct {
// Name is the descriptive name of the mutant.
Name Name
// Index is the mutant index.
//
// The mutant index is what is passed into the mutation environment
// variable, and is the basis for mutant definition by range.
Index Index
}
Mutant is an identifier for a particular mutant.
Since we only support a mutation testing setups with integer mutant identifiers, this is just uint64 for now.
func AnonMutant ¶
AnonMutant creates a mutant with index i, but no name.
func NamedMutant ¶
NamedMutant creates a named mutant with index i, operator operator and variant variant. If operator is empty, the variant will not be recorded.
func (*Mutant) SetIndexIfZero ¶
SetIndexIfZero sets this mutant's index to i if it is currently zero.
func (Mutant) String ¶
String gets a human-readable string representation of this mutant.
If a name is available, the string will contain it.
Example ¶
ExampleMutant_String is a runnable example for Mutant.String.
package main
import (
"fmt"
"github.com/c4-project/c4t/internal/mutation"
)
func main() {
fmt.Println(mutation.NamedMutant(42, "", 0))
fmt.Println(mutation.NamedMutant(56, "", 10))
fmt.Println(mutation.NamedMutant(12, "FOO", 0))
fmt.Println(mutation.NamedMutant(13, "BAR", 1))
}
Output: 42 56 FOO:12 BAR1:13
type MutantAnalysis ¶
type MutantAnalysis struct {
// Mutant contains the full record for this mutant.
Mutant Mutant
// Killed records whether this mutant was killed.
Killed bool
// Selections contains the per-selection analysis for this mutant.
Selections []SelectionAnalysis
}
MutantAnalysis is the type of individual mutant analyses.
func (*MutantAnalysis) AddSelection ¶
func (a *MutantAnalysis) AddSelection(sel SelectionAnalysis)
AddSelection adds sel to a's selection analyses.
type Name ¶
type Name struct {
// Operator is the name of the mutant operator, if given.
Operator string
// Variant is the index of this particular mutant within its operator.
Variant uint64
}
Name is a human-readable name for mutants.
type Range ¶
type Range struct {
// Operator is, if given, the name of the operator in this range.
Operator string `json:"operator" toml:"operator"`
// Start is the first mutant number to consider in this range.
Start Index `json:"start" toml:"start"`
// End is one past the last mutant number to consider in this range.
End Index `json:"end" toml:"end"`
}
Range defines an inclusive numeric range of mutant numbers to consider.
func (Range) IsEmpty ¶
IsEmpty gets whether this range defines no mutant numbers.
Example ¶
ExampleRange_IsEmpty is a runnable example for IsEmpty.
package main
import (
"fmt"
"github.com/c4-project/c4t/internal/mutation"
)
func main() {
fmt.Println("10..20:", mutation.Range{Start: 10, End: 20}.IsEmpty())
fmt.Println("10..10:", mutation.Range{Start: 10, End: 10}.IsEmpty())
fmt.Println("20..10:", mutation.Range{Start: 20, End: 10}.IsEmpty())
fmt.Println("10..11:", mutation.Range{Start: 10, End: 11}.IsEmpty())
}
Output: 10..20: false 10..10: true 20..10: true 10..11: false
func (Range) IsSingleton ¶
IsSingleton gets whether this range has exactly one item in it.
func (Range) Mutants ¶
Mutants expands a range into the slice of mutant numbers falling within it.
Example ¶
ExampleRange_Mutants is a runnable example for Range.
package main
import (
"fmt"
"github.com/c4-project/c4t/internal/mutation"
)
func main() {
fmt.Print("unnamed:")
for _, i := range (mutation.Range{Start: 10, End: 20}).Mutants() {
fmt.Printf(" %s", i)
}
fmt.Println()
fmt.Print("named: ")
for _, i := range (mutation.Range{Operator: "ABC", Start: 20, End: 23}).Mutants() {
fmt.Printf(" %s", i)
}
fmt.Println()
}
Output: unnamed: 10 11 12 13 14 15 16 17 18 19 named: ABC1:20 ABC2:21 ABC3:22
type SelectionAnalysis ¶
type SelectionAnalysis struct {
// Timespan represents the timespan at which the compilation finished.
Timespan timing.Span `json:"time_span"`
// NumHits is the number of times this compilation hit the mutant.
// If this is 0, the mutant was selected but never hit.
NumHits uint64 `json:"num_hits"`
// Status was the main status of the compilation, which determines whether the selection killed the mutant.
Status status.Status `json:"status"`
// HitBy is the name of the compilation that hit this mutant.
HitBy compilation.Name `json:"by"`
}
SelectionAnalysis represents one instance where a compilation selected a particular mutant.
func (SelectionAnalysis) Hit ¶
func (h SelectionAnalysis) Hit() bool
Hit gets whether this selection resulted in at least one hit.
func (SelectionAnalysis) Killed ¶
func (h SelectionAnalysis) Killed() bool
Killed gets whether this selection resulted in a kill (hit at least once and resulted in a flagged status).
Source Files
Directories