textsplitter

package
v0.0.110 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Mar 29, 2024 License: MIT Imports: 5 Imported by: 0

Documentation

Overview

Package textsplitter provides utilities for splitting and processing text.

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type BaseTextSplitter

type BaseTextSplitter struct {
	// contains filtered or unexported fields
}

func NewBaseTextSplitter

func NewBaseTextSplitter(splitTextFunc SplitTextFunc, optFns ...func(o *Options)) *BaseTextSplitter

func (*BaseTextSplitter) CreateDocuments

func (ts *BaseTextSplitter) CreateDocuments(texts []string, metadatas []map[string]any) ([]schema.Document, error)

func (*BaseTextSplitter) SplitDocuments

func (ts *BaseTextSplitter) SplitDocuments(docs []schema.Document) ([]schema.Document, error)

type CharacterTextSplitter

type CharacterTextSplitter struct {
	*BaseTextSplitter
	// contains filtered or unexported fields
}

func NewCharacterTextSplitter

func NewCharacterTextSplitter(optFns ...func(o *CharacterTextSplitterOptions)) *CharacterTextSplitter

type CharacterTextSplitterOptions

type CharacterTextSplitterOptions struct {
	Options
	Separator string
}

type LengthFunc

type LengthFunc func(text string) int

type Options

type Options struct {
	ChunkSize     int
	ChunkOverlap  int
	KeepSeparator bool
	LengthFunc    LengthFunc
}

type RecursiveCharacterTextSplitter

type RecursiveCharacterTextSplitter struct {
	*BaseTextSplitter
	// contains filtered or unexported fields
}

func NewRecusiveCharacterTextSplitter

func NewRecusiveCharacterTextSplitter(optFns ...func(o *RecursiveCharacterTextSplitterOptions)) *RecursiveCharacterTextSplitter

type RecursiveCharacterTextSplitterOptions

type RecursiveCharacterTextSplitterOptions struct {
	Options
	Separators []string
}

type SplitTextFunc

type SplitTextFunc func(text string) []string

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL