metrics

Documentation

Index

• Variables
• type AssessmentKey
  • func RegisterAssessmentKey(key string) AssessmentKey
• type Assessments
  • func CombineWithSymflowerFixAssessments(model Assessments, fixed Assessments) (combined Assessments)
  • func NewAssessments() Assessments
  • func (a Assessments) Add(x Assessments)
  • func (a Assessments) Award(key AssessmentKey)
  • func (a Assessments) AwardMultiple(key AssessmentKey, count uint64)
  • func (a Assessments) Equal(x Assessments) bool
  • func (a Assessments) String() string
  • func (a Assessments) StringCSV() (row []string)

Variables

var (
	// AssessmentKeyFilesExecuted holds the successfully executed files.
	AssessmentKeyFilesExecuted = RegisterAssessmentKey("files-executed")
	// AssessmentKeyFilesExecutedMaximumReachable holds the maximum theoretically reachable executed files.
	AssessmentKeyFilesExecutedMaximumReachable = RegisterAssessmentKey("files-executed-maximum-reachable")
	// AssessmentKeyProcessingTime holds the time in milliseconds that it took to complete the task.
	AssessmentKeyProcessingTime = RegisterAssessmentKey("processing-time")

	// AssessmentKeyCoverage counts execution coverage objects.
	AssessmentKeyCoverage = RegisterAssessmentKey("coverage")

	// AssessmentKeyTestsPassing holds the percentage of passing tests.
	AssessmentKeyTestsPassing = RegisterAssessmentKey("tests-passing")

	// AssessmentKeyResponseCharacterCount counts the number of characters of a response.
	AssessmentKeyResponseCharacterCount = RegisterAssessmentKey("response-character-count")
	// AssessmentKeyGenerateTestsForFileCharacterCount counts the number of characters of a generated test file.
	AssessmentKeyGenerateTestsForFileCharacterCount = RegisterAssessmentKey("generate-tests-for-file-character-count")

	// AssessmentKeyResponseNoError indicates that a model responded without error.
	AssessmentKeyResponseNoError = RegisterAssessmentKey("response-no-error")
	// AssessmentKeyResponseWithCode indicates that a model responded with code.
	AssessmentKeyResponseWithCode = RegisterAssessmentKey("response-with-code")
	// AssessmentKeyResponseNoExcess indicates that a model did not produce more content as requested.
	// TODO Infer if a model produced "too much" code. https://github.com/symflower/eval-dev-quality/issues/44
	AssessmentKeyResponseNoExcess = RegisterAssessmentKey("response-no-excess")
)

var (

	// AllAssessmentKeysStrings returns all registered assessment keys as strings.
	AllAssessmentKeysStrings []string
)

Types

type AssessmentKey

type AssessmentKey string

AssessmentKey defines a key for a numerical key-value assessment pair.

func RegisterAssessmentKey

func RegisterAssessmentKey(key string) AssessmentKey

RegisterAssessmentKey registers a new assessment key.

type Assessments

type Assessments map[AssessmentKey]uint64

Assessments holds a collection of numerical assessment metrics.

func CombineWithSymflowerFixAssessments

func CombineWithSymflowerFixAssessments(model Assessments, fixed Assessments) (combined Assessments)

CombineWithSymflowerFixAssessments combines the model assessments with the ones from "symflower fix".

func NewAssessments

func NewAssessments() Assessments

NewAssessments creates a new assessment collection.

func (Assessments) Add

func (a Assessments) Add(x Assessments)

Add adds the given assessment collection to the current one.

func (Assessments) Award

func (a Assessments) Award(key AssessmentKey)

Award yields a score point.

func (Assessments) AwardMultiple

func (a Assessments) AwardMultiple(key AssessmentKey, count uint64)

AwardMultiple yields multiple score points.

func (Assessments) Equal

func (a Assessments) Equal(x Assessments) bool

Equal checks if both assessment collections are equal.

func (Assessments) String

func (a Assessments) String() string

String returns a string representation of the metrics.

func (Assessments) StringCSV

func (a Assessments) StringCSV() (row []string)

StringCSV returns a CSV row string representation of the metrics.