transformers

Documentation

Index

• Constants
• Variables
• func DefaultDocumentTransformers(filetype string) (transformers []types.DocumentTransformer)
• func GetTransformer(name string) (dstypes.DocumentTransformer, error)
• type ExtraMetadata
  • func (e *ExtraMetadata) Name() string
  • func (e *ExtraMetadata) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)
• type FilterMarkdownDocsNoContent
  • func (f *FilterMarkdownDocsNoContent) Name() string
  • func (f *FilterMarkdownDocsNoContent) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)
• type GenericTransformer
  • func (g *GenericTransformer) Transform(ctx context.Context, docs []vs.Document) ([]vs.Document, error)
• type KeywordExtractor
  • func NewKeyWordExtractor(numKeywords int, llm llm.LLM) *KeywordExtractor
  • func (k *KeywordExtractor) Name() string
  • func (k *KeywordExtractor) Transform(ctx context.Context, docs []vs.Document) ([]vs.Document, error)
• type MetadataManipulation
• type MetadataManipulationOperator
• type MetadataManipulator
  • func (m *MetadataManipulator) Name() string
  • func (m *MetadataManipulator) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)

Constants

const ExtraMetadataName = "extra_metadata"

const FilterMarkdownDocsNoContentName = "filter_markdown_docs_no_content"

const KeywordExtractorName = "keywords"

const MetadataManipulatorName = "metadata"

Variables

var TransformerMap = map[string]dstypes.DocumentTransformer{
	ExtraMetadataName:               &ExtraMetadata{},
	FilterMarkdownDocsNoContentName: &FilterMarkdownDocsNoContent{},
	KeywordExtractorName:            &KeywordExtractor{},
	MetadataManipulatorName:         &MetadataManipulator{},
}

Functions

func DefaultDocumentTransformers

func DefaultDocumentTransformers(filetype string) (transformers []types.DocumentTransformer)

func GetTransformer

func GetTransformer(name string) (dstypes.DocumentTransformer, error)

Types

type ExtraMetadata

type ExtraMetadata struct {
	Metadata map[string]any
}

func (*ExtraMetadata) Name

func (e *ExtraMetadata) Name() string

func (*ExtraMetadata) Transform

func (e *ExtraMetadata) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)

type FilterMarkdownDocsNoContent

type FilterMarkdownDocsNoContent struct{}

FilterMarkdownDocsNoContent filters out Markdown documents with no content or only headings

TODO: this may be moved into the MarkdownTextSplitter

func (*FilterMarkdownDocsNoContent) Name

func (f *FilterMarkdownDocsNoContent) Name() string

func (*FilterMarkdownDocsNoContent) Transform

func (f *FilterMarkdownDocsNoContent) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)

type GenericTransformer

type GenericTransformer struct {
	TransformationFunc func(context.Context, []vs.Document) ([]vs.Document, error)
}

func (*GenericTransformer) Transform

func (g *GenericTransformer) Transform(ctx context.Context, docs []vs.Document) ([]vs.Document, error)

type KeywordExtractor

type KeywordExtractor struct {
	NumKeywords int
	LLM         llm.LLM
}

func NewKeyWordExtractor

func NewKeyWordExtractor(numKeywords int, llm llm.LLM) *KeywordExtractor

func (*KeywordExtractor) Name

func (k *KeywordExtractor) Name() string

func (*KeywordExtractor) Transform

func (k *KeywordExtractor) Transform(ctx context.Context, docs []vs.Document) ([]vs.Document, error)

type MetadataManipulation

type MetadataManipulation struct {
	Operator MetadataManipulationOperator `json:"operator,omitempty" mapstructure:"operator"`
	Key      string                       `json:"key,omitempty" mapstructure:"key"`
	Value    any                          `json:"value,omitempty" mapstructure:"value"`
}

type MetadataManipulationOperator

type MetadataManipulationOperator string

const (
	MetadataManipulationOperatorAdd    MetadataManipulationOperator = "add"
	MetadataManipulationOperatorUpdate MetadataManipulationOperator = "upsert"
	MetadataManipulationOperatorRemove MetadataManipulationOperator = "remove"
)

type MetadataManipulator

type MetadataManipulator struct {
	Manipulations []MetadataManipulation
}

func (*MetadataManipulator) Name

func (m *MetadataManipulator) Name() string

func (*MetadataManipulator) Transform

func (m *MetadataManipulator) Transform(_ context.Context, docs []vs.Document) ([]vs.Document, error)