Versions in this module Expand all Collapse all v1 v1.0.1 Nov 9, 2023 v1.0.0 Nov 8, 2023 Changes in this version + var ErrMismatchMetadatasAndText = errors.New("number of texts and metadatas does not match") + func CreateDocuments(textSplitter TextSplitter, texts []string, metadatas []map[string]any) ([]schema.Document, error) + func SplitDocuments(textSplitter TextSplitter, documents []schema.Document) ([]schema.Document, error) + type MarkdownTextSplitter struct + ChunkOverlap int + ChunkSize int + SecondSplitter TextSplitter + func NewMarkdownTextSplitter(opts ...Option) *MarkdownTextSplitter + func (sp MarkdownTextSplitter) SplitText(text string) ([]string, error) + type Option func(*Options) + func WithAllowedSpecial(allowedSpecial []string) Option + func WithChunkOverlap(chunkOverlap int) Option + func WithChunkSize(chunkSize int) Option + func WithDisallowedSpecial(disallowedSpecial []string) Option + func WithEncodingName(encodingName string) Option + func WithModelName(modelName string) Option + func WithSecondSplitter(secondSplitter TextSplitter) Option + func WithSeparators(separators []string) Option + type Options struct + AllowedSpecial []string + ChunkOverlap int + ChunkSize int + DisallowedSpecial []string + EncodingName string + ModelName string + SecondSplitter TextSplitter + Separators []string + func DefaultOptions() Options + type RecursiveCharacter struct + ChunkOverlap int + ChunkSize int + Separators []string + func NewRecursiveCharacter(opts ...Option) RecursiveCharacter + func (s RecursiveCharacter) SplitText(text string) ([]string, error) + type TextSplitter interface + SplitText func(text string) ([]string, error) + type TokenSplitter struct + AllowedSpecial []string + ChunkOverlap int + ChunkSize int + DisallowedSpecial []string + EncodingName string + ModelName string + func NewTokenSplitter(opts ...Option) TokenSplitter + func (s TokenSplitter) SplitText(text string) ([]string, error)