transformers

Documentation

Index

• Constants
• Variables
• func DefaultDocumentTransformers(filetype string) (transformers []types.DocumentTransformer)
• func GetTransformer(name string) (types.DocumentTransformer, error)
• type ExtraMetadata
  • func (e *ExtraMetadata) Name() string
  • func (e *ExtraMetadata) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)
• type FilterMarkdownDocsNoContent
  • func (f *FilterMarkdownDocsNoContent) Name() string
  • func (f *FilterMarkdownDocsNoContent) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)
• type GenericTransformer
  • func (g *GenericTransformer) Transform(ctx context.Context, docs []vs.Document) ([]vs.Document, error)
• type KeywordExtractor
  • func NewKeyWordExtractor(numKeywords int, llm llm.LLM) *KeywordExtractor
  • func (k *KeywordExtractor) Name() string
  • func (k *KeywordExtractor) Transform(ctx context.Context, docs []vs.Document) ([]vs.Document, error)

Constants

const ExtraMetadataName = "extra_metadata"

const FilterMarkdownDocsNoContentName = "filter_markdown_docs_no_content"

const KeywordExtractorName = "keywords"

Variables

var TransformerMap = map[string]types.DocumentTransformer{
	ExtraMetadataName:               &ExtraMetadata{},
	FilterMarkdownDocsNoContentName: &FilterMarkdownDocsNoContent{},
	KeywordExtractorName:            &KeywordExtractor{},
}

Functions

func DefaultDocumentTransformers

func DefaultDocumentTransformers(filetype string) (transformers []types.DocumentTransformer)

func GetTransformer

func GetTransformer(name string) (types.DocumentTransformer, error)

Types

type ExtraMetadata

type ExtraMetadata struct {
	Metadata map[string]any
}

func (*ExtraMetadata) Name

func (e *ExtraMetadata) Name() string

func (*ExtraMetadata) Transform

func (e *ExtraMetadata) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)

type FilterMarkdownDocsNoContent

type FilterMarkdownDocsNoContent struct{}

FilterMarkdownDocsNoContent filters out Markdown documents with no content or only headings

TODO: this may be moved into the MarkdownTextSplitter

func (*FilterMarkdownDocsNoContent) Name

func (f *FilterMarkdownDocsNoContent) Name() string

func (*FilterMarkdownDocsNoContent) Transform

func (f *FilterMarkdownDocsNoContent) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)

type GenericTransformer

type GenericTransformer struct {
	TransformationFunc func(context.Context, []vs.Document) ([]vs.Document, error)
}

func (*GenericTransformer) Transform

func (g *GenericTransformer) Transform(ctx context.Context, docs []vs.Document) ([]vs.Document, error)

type KeywordExtractor

type KeywordExtractor struct {
	NumKeywords int
	LLM         llm.LLM
}

func NewKeyWordExtractor

func NewKeyWordExtractor(numKeywords int, llm llm.LLM) *KeywordExtractor

func (*KeywordExtractor) Name

func (k *KeywordExtractor) Name() string

func (*KeywordExtractor) Transform

func (k *KeywordExtractor) Transform(ctx context.Context, docs []vs.Document) ([]vs.Document, error)