postprocessors

Documentation

Overview

Package postprocessors is basically the same as package transformers, but used at a different stage of the RAG pipeline

Index

• Constants
• Variables
• type CohereRerankPostprocessor
  • func (c *CohereRerankPostprocessor) Name() string
  • func (c *CohereRerankPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type ContentFilterPostprocessor
  • func (c *ContentFilterPostprocessor) Name() string
  • func (c *ContentFilterPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type ContentSubstringFilterPostprocessor
  • func (c *ContentSubstringFilterPostprocessor) Name() string
  • func (c *ContentSubstringFilterPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type Postprocessor
  • func GetPostprocessor(name string) (Postprocessor, error)
• type ReducePostprocessor
  • func (s *ReducePostprocessor) Name() string
  • func (s *ReducePostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type SimilarityPostprocessor
  • func (s *SimilarityPostprocessor) Name() string
  • func (s *SimilarityPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error
• type TransformerWrapper
  • func NewTransformerWrapper(transformer types.DocumentTransformer) *TransformerWrapper
  • func (t *TransformerWrapper) Name() string
  • func (t *TransformerWrapper) Transform(ctx context.Context, response *types.RetrievalResponse) error

Constants

const CohereRerankPostprocessorName = "cohere_rerank"

const ContentFilterPostprocessorName = "content_filter"

const ContentSubstringFilterPostprocessorName = "content_substring_filter"

const ReducePostprocessorName = "reduce"

const SimilarityPostprocessorName = "similarity"

Variables

var PostprocessorMap = map[string]Postprocessor{
	transformers.ExtraMetadataName:               NewTransformerWrapper(&transformers.ExtraMetadata{}),
	transformers.KeywordExtractorName:            NewTransformerWrapper(&transformers.KeywordExtractor{}),
	transformers.FilterMarkdownDocsNoContentName: NewTransformerWrapper(&transformers.FilterMarkdownDocsNoContent{}),
	SimilarityPostprocessorName:                  &SimilarityPostprocessor{},
	ContentSubstringFilterPostprocessorName:      &ContentSubstringFilterPostprocessor{},
	ContentFilterPostprocessorName:               &ContentFilterPostprocessor{},
	CohereRerankPostprocessorName:                &CohereRerankPostprocessor{},
	ReducePostprocessorName:                      &ReducePostprocessor{},
}

Types

type CohereRerankPostprocessor

type CohereRerankPostprocessor struct {
	ApiKey string `json:"apiKey" yaml:"apiKey"`
	Model  string
	TopN   int
}

func (*CohereRerankPostprocessor) Name

func (c *CohereRerankPostprocessor) Name() string

func (*CohereRerankPostprocessor) Transform

func (c *CohereRerankPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type ContentFilterPostprocessor

type ContentFilterPostprocessor struct {
	Question string // Question about the content, that can be answered with yes or no
	Include  bool   // Whether to include or exclude the documents for which the answer is yes
	LLM      llm.LLM
}

func (*ContentFilterPostprocessor) Name

func (c *ContentFilterPostprocessor) Name() string

func (*ContentFilterPostprocessor) Transform

func (c *ContentFilterPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type ContentSubstringFilterPostprocessor

type ContentSubstringFilterPostprocessor struct {
	Contains    []string
	NotContains []string
}

func (*ContentSubstringFilterPostprocessor) Name

func (c *ContentSubstringFilterPostprocessor) Name() string

func (*ContentSubstringFilterPostprocessor) Transform

func (c *ContentSubstringFilterPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type Postprocessor

type Postprocessor interface {
	Transform(ctx context.Context, response *types.RetrievalResponse) error
	Name() string
}

Postprocessor is similar to types.DocumentTransformer, but can take into account the retrieval query

func GetPostprocessor

func GetPostprocessor(name string) (Postprocessor, error)

type ReducePostprocessor

type ReducePostprocessor struct {
	TopK int
}

func (*ReducePostprocessor) Name

func (s *ReducePostprocessor) Name() string

func (*ReducePostprocessor) Transform

func (s *ReducePostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type SimilarityPostprocessor

type SimilarityPostprocessor struct {
	Threshold float32
}

func (*SimilarityPostprocessor) Name

func (s *SimilarityPostprocessor) Name() string

func (*SimilarityPostprocessor) Transform

func (s *SimilarityPostprocessor) Transform(ctx context.Context, response *types.RetrievalResponse) error

type TransformerWrapper

type TransformerWrapper struct {
	types.DocumentTransformer
}

func NewTransformerWrapper

func NewTransformerWrapper(transformer types.DocumentTransformer) *TransformerWrapper

func (*TransformerWrapper) Name

func (t *TransformerWrapper) Name() string

func (*TransformerWrapper) Transform

func (t *TransformerWrapper) Transform(ctx context.Context, response *types.RetrievalResponse) error