textsplitter

Documentation

Overview

Package textsplitter provides utilities for splitting and processing text.

Index

• type BaseTextSplitter
  • func NewBaseTextSplitter(splitTextFunc SplitTextFunc, optFns ...func(o *Options)) *BaseTextSplitter
  • func (ts *BaseTextSplitter) CreateDocuments(texts []string, metadatas []map[string]any) ([]schema.Document, error)
  • func (ts *BaseTextSplitter) SplitDocuments(docs []schema.Document) ([]schema.Document, error)
• type CharacterTextSplitter
  • func NewCharacterTextSplitter(optFns ...func(o *CharacterTextSplitterOptions)) *CharacterTextSplitter
• type CharacterTextSplitterOptions
• type LengthFunc
• type Options
• type RecursiveCharacterTextSplitter
  • func NewRecusiveCharacterTextSplitter(optFns ...func(o *RecursiveCharacterTextSplitterOptions)) *RecursiveCharacterTextSplitter
• type RecursiveCharacterTextSplitterOptions
• type SplitTextFunc

Types

type BaseTextSplitter

type BaseTextSplitter struct {
	// contains filtered or unexported fields
}

func NewBaseTextSplitter

func NewBaseTextSplitter(splitTextFunc SplitTextFunc, optFns ...func(o *Options)) *BaseTextSplitter

func (*BaseTextSplitter) CreateDocuments

func (ts *BaseTextSplitter) CreateDocuments(texts []string, metadatas []map[string]any) ([]schema.Document, error)

func (*BaseTextSplitter) SplitDocuments

func (ts *BaseTextSplitter) SplitDocuments(docs []schema.Document) ([]schema.Document, error)

type CharacterTextSplitter

type CharacterTextSplitter struct {
	*BaseTextSplitter
	// contains filtered or unexported fields
}

func NewCharacterTextSplitter

func NewCharacterTextSplitter(optFns ...func(o *CharacterTextSplitterOptions)) *CharacterTextSplitter

type CharacterTextSplitterOptions

type CharacterTextSplitterOptions struct {
	Options
	Separator string
}

type LengthFunc

type LengthFunc func(text string) int

type Options

type Options struct {
	ChunkSize     int
	ChunkOverlap  int
	KeepSeparator bool
	LengthFunc    LengthFunc
}

type RecursiveCharacterTextSplitter

type RecursiveCharacterTextSplitter struct {
	*BaseTextSplitter
	// contains filtered or unexported fields
}

func NewRecusiveCharacterTextSplitter

func NewRecusiveCharacterTextSplitter(optFns ...func(o *RecursiveCharacterTextSplitterOptions)) *RecursiveCharacterTextSplitter

type RecursiveCharacterTextSplitterOptions

type RecursiveCharacterTextSplitterOptions struct {
	Options
	Separators []string
}

type SplitTextFunc

type SplitTextFunc func(text string) []string