v1alpha1

Documentation

Overview

Copyright 2024 KubeAGI.

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Package v1alpha1 contains API Schema definitions for the evaluation.arcadia v1alpha1 API group +kubebuilder:object:generate=true +groupName=evaluation.arcadia.kubeagi.k8s.com.cn

Index

• Constants
• Variables
• func RAGSpecChanged(a, b RAGSpec) bool
• func RagStatusChanged(a, b RAGStatus) bool
• type Dataset
  • func (in *Dataset) DeepCopy() *Dataset
  • func (in *Dataset) DeepCopyInto(out *Dataset)
• type Metric
  • func (in *Metric) DeepCopy() *Metric
  • func (in *Metric) DeepCopyInto(out *Metric)
• type MetricsKind
• type Parameter
  • func (in *Parameter) DeepCopy() *Parameter
  • func (in *Parameter) DeepCopyInto(out *Parameter)
• type RAG
  • func (in *RAG) DeepCopy() *RAG
  • func (in *RAG) DeepCopyInto(out *RAG)
  • func (in *RAG) DeepCopyObject() runtime.Object
• type RAGList
  • func (in *RAGList) DeepCopy() *RAGList
  • func (in *RAGList) DeepCopyInto(out *RAGList)
  • func (in *RAGList) DeepCopyObject() runtime.Object
• type RAGPhase
  • func RagStatus(rag *RAG) (string, RAGPhase, string)
• type RAGSpec
  • func (in *RAGSpec) DeepCopy() *RAGSpec
  • func (in *RAGSpec) DeepCopyInto(out *RAGSpec)
• type RAGStatus
  • func (in *RAGStatus) DeepCopy() *RAGStatus
  • func (in *RAGStatus) DeepCopyInto(out *RAGStatus)
• type Report
  • func (in *Report) DeepCopy() *Report
  • func (in *Report) DeepCopyInto(out *Report)

Constants

const (
	EvaluationJobLabels        = Group + "/rag"
	EvaluationApplicationLabel = Group + "/application"
)

Define RAG common structure and variables

const (
	ING      = "ing" // evaluating
	COMPLETE = "complete"
	FAILED   = "failed"
	SUSPEND  = "suspend"
)

const (
	Group   = "evaluation.arcadia.kubeagi.k8s.com.cn"
	Version = "v1alpha1"
)

Variables

var (
	// GroupVersion is group version used to register these objects
	GroupVersion = schema.GroupVersion{Group: Group, Version: Version}

	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}

	// AddToScheme adds the types in this group-version to the given scheme.
	AddToScheme = SchemeBuilder.AddToScheme
)

Functions

func RAGSpecChanged

func RAGSpecChanged(a, b RAGSpec) bool

func RagStatusChanged

func RagStatusChanged(a, b RAGStatus) bool

Types

type Dataset

type Dataset struct {
	// From defines the source which provides this QA Files for test dataset
	// Only `VersionedDataset` allowed
	Source *basev1alpha1.TypedObjectReference `json:"source,omitempty"`
	// Files retrieved from Source and used in this testdataset
	// - For file with tag `object_type: QA`, will be used directly
	// - TODO: For file without special tags, will use `QAGenerationChain` to generate QAs (Not Supported Yet)
	Files []string `json:"files,omitempty"`
}

Dataset stands for the files used to generate ragas test dataset

func (*Dataset) DeepCopy

func (in *Dataset) DeepCopy() *Dataset

DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Dataset.

func (*Dataset) DeepCopyInto

func (in *Dataset) DeepCopyInto(out *Dataset)

DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.

type Metric

type Metric struct {
	// Kind of this Metric
	Kind MetricsKind `json:"kind,omitempty"`

	// Parameters in this Metrics
	Parameters []Parameter `json:"parameters,omitempty"`

	// ToleranceThreshbold on this Metric
	// If the evaluation score is smaller than this tolerance threshold,we treat this RAG solution as `Bad`
	ToleranceThreshbold int `json:"tolerance_threshold,omitempty"`
}

func (*Metric) DeepCopy

func (in *Metric) DeepCopy() *Metric

DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Metric.

func (*Metric) DeepCopyInto

func (in *Metric) DeepCopyInto(out *Metric)

DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.

type MetricsKind

type MetricsKind string

const (
	// AnswerRelevancy in ragas https://docs.ragas.io/en/stable/concepts/metrics/answer_relevance.html
	// Scores the relevancy of the answer according to the given question.
	AnswerRelevancy MetricsKind = "answer_relevancy"

	// AnswerSimilarity in ragas https://docs.ragas.io/en/stable/concepts/metrics/semantic_similarity.html
	// Scores the semantic similarity of ground truth with generated answer.
	AnswerSimilarity MetricsKind = "answer_similarity"

	// AnswerCorrectness in ragas https://docs.ragas.io/en/stable/concepts/metrics/answer_correctness.html
	// Measures answer correctness compared to ground truth as a combination(Weighted) of
	// - factuality
	// - semantic similarity
	AnswerCorrectness MetricsKind = "answer_correctness"

	// Faithfulness in ragas https://docs.ragas.io/en/stable/concepts/metrics/faithfulness.html
	// Scores the factual consistency of the generated answer against the given context.
	Faithfulness MetricsKind = "faithfulness"

	// ContextPrecision in ragas https://docs.ragas.io/en/stable/concepts/metrics/context_precision.html
	// Average Precision is a metric that evaluates whether all of the relevant items selected by the model are ranked higher or not.
	ContextPrecision MetricsKind = "context_precision"

	// ContextRelevancy in ragas https://docs.ragas.io/en/stable/concepts/metrics/context_relevancy.html
	// Gauges the relevancy of the retrieved context
	ContextRelevancy MetricsKind = "context_relevancy"

	// ContextRecall in ragas https://docs.ragas.io/en/stable/concepts/metrics/context_recall.html
	// Estimates context recall by estimating TP and FN using annotated answer and retrieved context.
	ContextRecall MetricsKind = "context_recall"

	// AspectCritique in ragas https://docs.ragas.io/en/stable/concepts/metrics/critique.html
	// Designed to assess submissions based on predefined aspects such as harmlessness and correctness
	// SUPPORTED_ASPECTS = [ harmfulness, maliciousness, coherence, correctness, conciseness, ]
	AspectCritique MetricsKind = "aspect_critique"
)

type Parameter

type Parameter struct {
	Key   string `json:"key,omitempty"`
	Value string `json:"value,omitempty"`
}

Parameter to metrics which is a key-value pair

func (*Parameter) DeepCopy

func (in *Parameter) DeepCopy() *Parameter

DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Parameter.

func (*Parameter) DeepCopyInto

func (in *Parameter) DeepCopyInto(out *Parameter)

DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.

type RAG

type RAG struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   RAGSpec   `json:"spec,omitempty"`
	Status RAGStatus `json:"status,omitempty"`
}

RAG is the Schema for the rags API

func (*RAG) DeepCopy

func (in *RAG) DeepCopy() *RAG

DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAG.

func (*RAG) DeepCopyInto

func (in *RAG) DeepCopyInto(out *RAG)

DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.

func (*RAG) DeepCopyObject

func (in *RAG) DeepCopyObject() runtime.Object

DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.

type RAGList

type RAGList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitempty"`
	Items           []RAG `json:"items"`
}

RAGList contains a list of RAG

func (*RAGList) DeepCopy

func (in *RAGList) DeepCopy() *RAGList

DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAGList.

func (*RAGList) DeepCopyInto

func (in *RAGList) DeepCopyInto(out *RAGList)

DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.

func (*RAGList) DeepCopyObject

func (in *RAGList) DeepCopyObject() runtime.Object

DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.

type RAGPhase

type RAGPhase string

const (
	InitPvcPhase           RAGPhase = "init"
	DownloadFilesPhase     RAGPhase = "download"
	GenerateTestFilesPhase RAGPhase = "generate"
	JudgeLLMPhase          RAGPhase = "judge"
	UploadFilesPhase       RAGPhase = "upload"
	CompletePhase          RAGPhase = "complete"
)

func RagStatus

func RagStatus(rag *RAG) (string, RAGPhase, string)

type RAGSpec

type RAGSpec struct {
	// CommonSpec
	basev1alpha1.CommonSpec `json:",inline"`

	// Application(required) defines the target of this RAG evaluation
	Application *basev1alpha1.TypedObjectReference `json:"application"`

	// Datasets defines the dataset which will be used to generate test datasets
	Datasets []Dataset `json:"datasets"`

	// JudgeLLM(required) defines the judge which is a LLM to evaluate RAG application against test dataset
	JudgeLLM *basev1alpha1.TypedObjectReference `json:"judge_llm"`

	// Metrics that this rag evaluation will do
	Metrics []Metric `json:"metrics"`

	// Report defines the evaluation report configurations
	Report Report `json:"report,omitempty"`

	// Storage storage must be provided and data needs to be saved throughout the evaluation phase.
	Storage *corev1.PersistentVolumeClaimSpec `json:"storage"`

	// ServiceAccountName define the user when the job is run
	// +kubebuilder:default=default
	ServiceAccountName string `json:"serviceAccountName,omitempty"`

	// Suspend suspension of the evaluation process
	// +kubebuilder:default=false
	Suspend bool `json:"suspend,omitempty"`
}

RAGSpec defines the desired state of RAG

func (*RAGSpec) DeepCopy

func (in *RAGSpec) DeepCopy() *RAGSpec

DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAGSpec.

func (*RAGSpec) DeepCopyInto

func (in *RAGSpec) DeepCopyInto(out *RAGSpec)

DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.

type RAGStatus

type RAGStatus struct {
	// CompletionTime Evaluation completion time
	CompletionTime *metav1.Time `json:"completionTime,omitempty"`

	// Phase evaluation current stage,
	// init,download,generate,judge,upload,complete
	Phase RAGPhase `json:"phase,omitempty"`

	// Conditions show the status of the job in the current stage
	Conditions []v1.JobCondition `json:"conditions,omitempty"`
}

RAGStatus defines the observed state of RAG

func (*RAGStatus) DeepCopy

func (in *RAGStatus) DeepCopy() *RAGStatus

DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RAGStatus.

func (*RAGStatus) DeepCopyInto

func (in *RAGStatus) DeepCopyInto(out *RAGStatus)

DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.

type Report

type Report struct {
}

Report is the summarization of evaluation

func (*Report) DeepCopy

func (in *Report) DeepCopy() *Report

DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Report.

func (*Report) DeepCopyInto

func (in *Report) DeepCopyInto(out *Report)

DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.