spliter

Documentation

Index

• Constants
• type Spliter
  • func NewTextSpliter(log logger.Logger, chunkSize int, chunkOverlap int, separator string) Spliter
• type TextSpliter
  • func (t *TextSpliter) Merge(elements []vector.Element) []vector.Element
  • func (t *TextSpliter) Split(text string) []string

Constants

const (
	DefaultChunkSize    = 4000
	DefaultChunkOverlap = 200
)

Types

type Spliter

type Spliter interface {
	Split(text string) []string
	Merge(elements []vector.Element) []vector.Element
}

func NewTextSpliter

func NewTextSpliter(log logger.Logger, chunkSize int, chunkOverlap int, separator string) Spliter

type TextSpliter

type TextSpliter struct {
	// contains filtered or unexported fields
}

func (*TextSpliter) Merge

func (t *TextSpliter) Merge(elements []vector.Element) []vector.Element

func (*TextSpliter) Split

func (t *TextSpliter) Split(text string) []string