evaluation

Documentation

Overview

Copyright 2024 KubeAGI.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Index

• Constants
• func DownloadJob(instance *evav1alpha1.RAG) (*batchv1.Job, error)
• func GenTestDataJob(instance *evav1alpha1.RAG) (*batchv1.Job, error)
• func JudgeJobGenerator(ctx context.Context, c client.Client) func(*evav1alpha1.RAG) (*batchv1.Job, error)
• func PhaseJobName(instance *evav1alpha1.RAG, phase evav1alpha1.RAGPhase) string
• func UploadJobGenerator(ctx context.Context, client client.Client) func(*evav1alpha1.RAG) (*batchv1.Job, error)
• type CSVOutput
  • func (csv *CSVOutput) Output(row RagasDataRow) error
• type GenOptions
  • func WithGroundTruthsColumn(groundTruthsColumn string) GenOptions
  • func WithMaxContextLength(maxContextLength int) GenOptions
  • func WithOutput(output Output) GenOptions
  • func WithQuestionColumn(questionColumn string) GenOptions
  • func WithWriteHeader(writeHeader bool) GenOptions
• type Output
• type PrintOutput
  • func (print *PrintOutput) Output(row RagasDataRow) error
• type RagasDataRow
• type RagasDatasetGenerator
  • func NewRagasDatasetGenerator(ctx context.Context, cli client.Client, app *v1alpha1.Application, ...) (*RagasDatasetGenerator, error)
  • func (eval *RagasDatasetGenerator) Generate(ctx context.Context, csvData io.Reader, genOptions ...GenOptions) error

Constants

const (

	// The clusterrolebinding required for the rag evaluation process is ragas-eval-clusterrolebinding by default,
	// and can be changed via environment variable RAG_EVAL_CLUSTERROLEBINDING.
	RAGClusterRoleBindingEnv = "RAG_EVAL_CLUSTERROLEBINDING"
	RAGJobClusterRoleBinding = "ragas-eval-clusterrolebinding"

	// The serviceaccount used by the job during rag evaluation, which is ragas-eval-sa by default,
	// can be changed via the environment variable RAG_EVAL_SERVICEACCOUNT.
	RAGServiceAccountEnv = "RAG_EVAL_SERVICEACCOUNT"
	RAGJobServiceAccount = "ragas-eval-sa"
)

Functions

func DownloadJob

func DownloadJob(instance *evav1alpha1.RAG) (*batchv1.Job, error)

func GenTestDataJob

func GenTestDataJob(instance *evav1alpha1.RAG) (*batchv1.Job, error)

func JudgeJobGenerator

func JudgeJobGenerator(ctx context.Context, c client.Client) func(*evav1alpha1.RAG) (*batchv1.Job, error)

func PhaseJobName

func PhaseJobName(instance *evav1alpha1.RAG, phase evav1alpha1.RAGPhase) string

func UploadJobGenerator

func UploadJobGenerator(ctx context.Context, client client.Client) func(*evav1alpha1.RAG) (*batchv1.Job, error)

Types

type CSVOutput

type CSVOutput struct {
	W *csv.Writer
}

CSVOutput writes row to csv

func (*CSVOutput) Output

func (csv *CSVOutput) Output(row RagasDataRow) error

Output a row to csv

type GenOptions

type GenOptions func(*genOptions)

func WithGroundTruthsColumn

func WithGroundTruthsColumn(groundTruthsColumn string) GenOptions

func WithMaxContextLength

func WithMaxContextLength(maxContextLength int) GenOptions

func WithOutput

func WithOutput(output Output) GenOptions

func WithQuestionColumn

func WithQuestionColumn(questionColumn string) GenOptions

func WithWriteHeader

func WithWriteHeader(writeHeader bool) GenOptions

type Output

type Output interface {
	Output(RagasDataRow) error
}

type PrintOutput

type PrintOutput struct{}

PrintOutput

func (*PrintOutput) Output

func (print *PrintOutput) Output(row RagasDataRow) error

Output this row to standard output

type RagasDataRow

type RagasDataRow struct {
	// Question by QAGeneration or manually input
	Question string `json:"question"`
	// GroundTruths by QAGeneration or manually input
	GroundTruths []string `json:"ground_truths"`
	// Contexts by similarity search to knowledgebase
	Contexts []string `json:"contexts"`
	// Answer by Application
	Answer string `json:"answer"`
	// Latency Q&A generation duration
	Latency string `json:"latency"`
}

RagasDataRow which adapts to the Ragas evaluation framework

type RagasDatasetGenerator

type RagasDatasetGenerator struct {
	// contains filtered or unexported fields
}

RagasDatasetGenerator generates datasets which adapts to the ragas framework

func NewRagasDatasetGenerator

func NewRagasDatasetGenerator(ctx context.Context, cli client.Client, app *v1alpha1.Application, genOptions ...GenOptions) (*RagasDatasetGenerator, error)

func (*RagasDatasetGenerator) Generate

func (eval *RagasDatasetGenerator) Generate(ctx context.Context, csvData io.Reader, genOptions ...GenOptions) error

Generate a test dataset from a file(csv)