postprocessors

Documentation

Overview

Package postprocessors is basically the same as package transformers, but used at a different stage of the RAG pipeline

Index

• Constants
• Variables
• type BM25Postprocessor
  • func (c *BM25Postprocessor) Name() string
  • func (c *BM25Postprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type CohereRerankPostprocessor
  • func (c *CohereRerankPostprocessor) Name() string
  • func (c *CohereRerankPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type ContentFilterPostprocessor
  • func (c *ContentFilterPostprocessor) Name() string
  • func (c *ContentFilterPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type ContentSubstringFilterPostprocessor
  • func (c *ContentSubstringFilterPostprocessor) Name() string
  • func (c *ContentSubstringFilterPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type Postprocessor
  • func GetPostprocessor(name string) (Postprocessor, error)
• type ReducePostprocessor
  • func (s *ReducePostprocessor) Name() string
  • func (s *ReducePostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type SimilarityPostprocessor
  • func (s *SimilarityPostprocessor) Name() string
  • func (s *SimilarityPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type TransformerWrapper
  • func NewTransformerWrapper(transformer types.DocumentTransformer) *TransformerWrapper
  • func (t *TransformerWrapper) Decode(cfg map[string]any) error
  • func (t *TransformerWrapper) Name() string
  • func (t *TransformerWrapper) Transform(ctx context.Context, response *types.RetrievalResponse) error

Constants

const BM25PostprocessorName = "bm25"

const CohereRerankPostprocessorName = "cohere_rerank"

const ContentFilterPostprocessorName = "content_filter"

const ContentSubstringFilterPostprocessorName = "content_substring_filter"

const ReducePostprocessorName = "reduce"

const SimilarityPostprocessorName = "similarity"

Variables

var PostprocessorMap = map[string]Postprocessor{
	transformers.ExtraMetadataName:               NewTransformerWrapper(&transformers.ExtraMetadata{}),
	transformers.KeywordExtractorName:            NewTransformerWrapper(&transformers.KeywordExtractor{}),
	transformers.FilterMarkdownDocsNoContentName: NewTransformerWrapper(&transformers.FilterMarkdownDocsNoContent{}),
	transformers.MetadataManipulatorName:         NewTransformerWrapper(&transformers.MetadataManipulator{}),
	SimilarityPostprocessorName:                  &SimilarityPostprocessor{},
	ContentSubstringFilterPostprocessorName:      &ContentSubstringFilterPostprocessor{},
	ContentFilterPostprocessorName:               &ContentFilterPostprocessor{},
	CohereRerankPostprocessorName:                &CohereRerankPostprocessor{},
	ReducePostprocessorName:                      &ReducePostprocessor{},
	BM25PostprocessorName:                        &BM25Postprocessor{},
}

Types

type BM25Postprocessor

type BM25Postprocessor struct {
	TopN         int
	SparseWeight float64 // How to weight the BM25 scores against the similarity scores from dense vector search

	K1 float64 // K1 should be between 1.2 and 2 - controls term frequency saturation
	B  float64 // B should be around 0.75 - controls the influence of document length normalization

	CleanStopWords []string // list of stopwords to remove from the documents - if empty, no stopwords are removed, if only "auto" is present, the language is detected automatically
}

func (*BM25Postprocessor) Name

func (c *BM25Postprocessor) Name() string

func (*BM25Postprocessor) Transform

func (c *BM25Postprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type CohereRerankPostprocessor

type CohereRerankPostprocessor struct {
	ApiKey string `json:"apiKey" yaml:"apiKey"`
	Model  string
	TopN   int
}

func (*CohereRerankPostprocessor) Name

func (c *CohereRerankPostprocessor) Name() string

func (*CohereRerankPostprocessor) Transform

func (c *CohereRerankPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type ContentFilterPostprocessor

type ContentFilterPostprocessor struct {
	Question string // Question about the content, that can be answered with yes or no
	Include  bool   // Whether to include or exclude the documents for which the answer is yes
	LLM      llm.LLM
}

func (*ContentFilterPostprocessor) Name

func (c *ContentFilterPostprocessor) Name() string

func (*ContentFilterPostprocessor) Transform

func (c *ContentFilterPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type ContentSubstringFilterPostprocessor

type ContentSubstringFilterPostprocessor struct {
	Contains    []string
	NotContains []string
}

func (*ContentSubstringFilterPostprocessor) Name

func (c *ContentSubstringFilterPostprocessor) Name() string

func (*ContentSubstringFilterPostprocessor) Transform

func (c *ContentSubstringFilterPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type Postprocessor

type Postprocessor interface {
	Transform(ctx context.Context, response *types.RetrievalResponse) error
	Name() string
}

Postprocessor is similar to types.DocumentTransformer, but can take into account the retrieval query

func GetPostprocessor

func GetPostprocessor(name string) (Postprocessor, error)

type ReducePostprocessor

type ReducePostprocessor struct {
	TopK int
}

func (*ReducePostprocessor) Name

func (s *ReducePostprocessor) Name() string

func (*ReducePostprocessor) Transform

func (s *ReducePostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type SimilarityPostprocessor

type SimilarityPostprocessor struct {
	Threshold float32
	KeepMin   int // KeepMin the top n documents, regardless of the threshold
}

func (*SimilarityPostprocessor) Name

func (s *SimilarityPostprocessor) Name() string

func (*SimilarityPostprocessor) Transform

func (s *SimilarityPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type TransformerWrapper

type TransformerWrapper struct {
	DocumentTransformer types.DocumentTransformer
}

func NewTransformerWrapper

func NewTransformerWrapper(transformer types.DocumentTransformer) *TransformerWrapper

func (*TransformerWrapper) Decode

func (t *TransformerWrapper) Decode(cfg map[string]any) error

func (*TransformerWrapper) Name

func (t *TransformerWrapper) Name() string

func (*TransformerWrapper) Transform

func (t *TransformerWrapper) Transform(ctx context.Context, response *types.RetrievalResponse) error