textsplitter

package
v0.0.17 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Jun 21, 2023 License: MIT Imports: 5 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type BaseTextSplitter

type BaseTextSplitter struct {
	// contains filtered or unexported fields
}

func NewBaseTextSplitter

func NewBaseTextSplitter(splitTextFunc SplitTextFunc, optFns ...func(o *Options)) *BaseTextSplitter

func (*BaseTextSplitter) CreateDocuments

func (ts *BaseTextSplitter) CreateDocuments(texts []string, metadatas []map[string]any) ([]schema.Document, error)

func (*BaseTextSplitter) SplitDocuments

func (ts *BaseTextSplitter) SplitDocuments(docs []schema.Document) ([]schema.Document, error)

type CharacterTextSplitter

type CharacterTextSplitter struct {
	*BaseTextSplitter
	// contains filtered or unexported fields
}

func NewCharacterTextSplitter

func NewCharacterTextSplitter(optFns ...func(o *CharacterTextSplitterOptions)) *CharacterTextSplitter

type CharacterTextSplitterOptions

type CharacterTextSplitterOptions struct {
	Options
	Separator string
}

type LengthFunc

type LengthFunc func(text string) int

type Options

type Options struct {
	ChunkSize     int
	ChunkOverlap  int
	KeepSeparator bool
	LengthFunc    LengthFunc
}

type RecursiveCharacterTextSplitter

type RecursiveCharacterTextSplitter struct {
	*BaseTextSplitter
	// contains filtered or unexported fields
}

func NewRecusiveCharacterTextSplitter

func NewRecusiveCharacterTextSplitter(optFns ...func(o *RecursiveCharacterTextSplitterOptions)) *RecursiveCharacterTextSplitter

type RecursiveCharacterTextSplitterOptions

type RecursiveCharacterTextSplitterOptions struct {
	Options
	Separators []string
}

type SplitTextFunc

type SplitTextFunc func(text string) []string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL