extractors

package
v0.13.0-beta.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 4, 2023 License: Apache-2.0 Imports: 18 Imported by: 0

Documentation

Index

Constants

View Source 
const DefaultPoolBuffer = 1000
View Source 
const DefaultPoolSize = 2
View Source 
const DefaultTaskChunkSize = 1000
View Source 
const SummaryMaxOutputTokens = 1024

Variables

View Source 
var IntentStringRegex = regexp.MustCompile(`(?i)^\s*intent\W+\s+`)

Functions

func Initialize 

func Initialize(appState *models.AppState)

Types

type BaseExtractor 

type BaseExtractor struct {
	// contains filtered or unexported fields
}

BaseExtractor is the base implementation of an Extractor

type DocEmbeddingProcessor added in v0.9.0 

type DocEmbeddingProcessor struct {
	EmbeddingTaskCh   chan []models.DocEmbeddingTask
	EmbeddingUpdateCh chan<- []models.DocEmbeddingUpdate
	Pool              *pond.WorkerPool
	PoolSize          int
	PoolBuffer        int
	ChunkSize         int
	// contains filtered or unexported fields
}

DocEmbeddingProcessor is a processor for embedding documents

func NewDocEmbeddingProcessor added in v0.9.0 

func NewDocEmbeddingProcessor(
	appState *models.AppState,
	embeddingTaskCh chan []models.DocEmbeddingTask,
	embeddingUpdateCh chan []models.DocEmbeddingUpdate,
) *DocEmbeddingProcessor

NewDocEmbeddingProcessor creates a new DocEmbeddingProcessor

func (*DocEmbeddingProcessor) Run added in v0.9.0 

func (ep *DocEmbeddingProcessor) Run(
	ctx context.Context,
) error

Run starts the DocEmbeddingProcessor. It is safe to call this multiple times. It will only start the processor once.

type EmbeddingExtractor 

type EmbeddingExtractor struct {
	BaseExtractor
}

func NewEmbeddingExtractor 

func NewEmbeddingExtractor() *EmbeddingExtractor

func (*EmbeddingExtractor) Extract 

func (ee *EmbeddingExtractor) Extract(
	ctx context.Context,
	appState *models.AppState,
	messageEvent *models.MessageEvent,
) error

func (*EmbeddingExtractor) Notify 

func (ee *EmbeddingExtractor) Notify(
	ctx context.Context,
	appState *models.AppState,
	messageEvents *models.MessageEvent,
) error

type EntityExtractor added in v0.5.0 

type EntityExtractor struct {
	BaseExtractor
}

func NewEntityExtractor added in v0.5.0 

func NewEntityExtractor() *EntityExtractor

func (*EntityExtractor) Extract added in v0.5.0 

func (ee *EntityExtractor) Extract(
	ctx context.Context,
	appState *models.AppState,
	messageEvent *models.MessageEvent,
) error

func (*EntityExtractor) Notify added in v0.5.0 

func (ee *EntityExtractor) Notify(
	ctx context.Context,
	appState *models.AppState,
	messageEvents *models.MessageEvent,
) error

type ExtractorError 

type ExtractorError struct {
	// contains filtered or unexported fields
}

func NewExtractorError 

func NewExtractorError(message string, originalError error) *ExtractorError

func (*ExtractorError) Error 

func (e *ExtractorError) Error() string

type IntentExtractor added in v0.7.0 

type IntentExtractor struct {
	BaseExtractor
}

func NewIntentExtractor added in v0.7.0 

func NewIntentExtractor() *IntentExtractor

func (*IntentExtractor) Extract added in v0.7.0 

func (ee *IntentExtractor) Extract(
	ctx context.Context,
	appState *models.AppState,
	messageEvent *models.MessageEvent,
) error

func (*IntentExtractor) Notify added in v0.7.0 

func (ee *IntentExtractor) Notify(
	ctx context.Context,
	appState *models.AppState,
	messageEvent *models.MessageEvent,
) error

type IntentPromptTemplateData added in v0.7.0 

type IntentPromptTemplateData struct {
	Input string
}

type SummaryExtractor 

type SummaryExtractor struct {
	BaseExtractor
}

func NewSummaryExtractor 

func NewSummaryExtractor() *SummaryExtractor

func (*SummaryExtractor) Extract 

func (se *SummaryExtractor) Extract(
	ctx context.Context,
	appState *models.AppState,
	messageEvent *models.MessageEvent,
) error

Extract gets a list of messages created since the last SummaryPoint, determines if the message count exceeds the configured message window, and if so: - determines the new SummaryPoint index, which will one message older than message_window / 2 - summarizes the messages from this new SummaryPoint to the oldest message not yet Summarized.

When summarizing, it adds context from these messages to an existing summary if there is one.

func (*SummaryExtractor) Notify 

func (se *SummaryExtractor) Notify(
	ctx context.Context,
	appState *models.AppState,
	messageEvents *models.MessageEvent,
) error

type SummaryPromptTemplateData 

type SummaryPromptTemplateData struct {
	PrevSummary    string
	MessagesJoined string
}

type TokenCountExtractor 

type TokenCountExtractor struct {
	BaseExtractor
	// contains filtered or unexported fields
}

func NewTokenCountExtractor 

func NewTokenCountExtractor() *TokenCountExtractor

func (*TokenCountExtractor) Extract 

func (ee *TokenCountExtractor) Extract(
	ctx context.Context,
	appState *models.AppState,
	messageEvents *models.MessageEvent,
) error

func (*TokenCountExtractor) Notify 

func (ee *TokenCountExtractor) Notify(
	ctx context.Context,
	appState *models.AppState,
	messageEvents *models.MessageEvent,
) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.