content

Documentation

Index

• func ChunkText(text string, chunkSize, overlap int) []string
• func CreateChunkContext(wholeDocumentContent string, chunk Chunk, ollamaUrl, contextualModel string, ...) (string, error)
• func CreateChunkContextWithPromptTemplate(promptTpl, wholeDocumentContent string, chunk Chunk, ...) (string, error)
• func FindFiles(dirPath string, ext string) ([]string, error)
• func ForEachFile(dirPath string, ext string, callback func(string) error) ([]string, error)
• func GenerateContextFromDocs(docs []string) string
• func GetArrayOfContentFiles(dirPath string, ext string) ([]string, error)
• func GetMapOfContentFiles(dirPath string, ext string) (map[string]string, error)
• func InterpolateString(str string, vars interface{}) (string, error)
• func ReadTextFile(path string) (string, error)
• func SplitAsciiDocBySections(content string) []string
• func SplitHTMLBySections(content string) ([]string, error)
• func SplitMarkdownByLevelSections(content string, level int) []string
• func SplitMarkdownBySections(content string) []string
• func SplitTextWithDelimiter(text string, delimiter string) []string
• func SplitTextWithRegex(text string, regexDelimiter string) []string
• func WriteTextFile(path, content string) error
• type Chunk
  • func ParseMarkdown(content string) []*Chunk
  • func ParseMarkdownWithHierarchy(content string) []Chunk
  • func ParseMarkdownWithLineage(content string) []Chunk

Functions

func ChunkText

func ChunkText(text string, chunkSize, overlap int) []string

ChunkText takes a text string and divides it into chunks of a specified size with a given overlap. It returns a slice of strings, where each string represents a chunk of the original text.

Parameters:

• text: The input text to be chunked.
• chunkSize: The size of each chunk.
• overlap: The amount of overlap between consecutive chunks.

Returns:

• []string: A slice of strings representing the chunks of the original text.

func CreateChunkContext

func CreateChunkContext(wholeDocumentContent string, chunk Chunk, ollamaUrl, contextualModel string, options llm.Options) (string, error)

CreateChunkContext generates a succinct context for a given chunk within the whole document content. This context is intended to improve search retrieval of the chunk.

Parameters:

• wholeDocumentContent: The entire content of the document as a string.
• chunk: The specific chunk of the document for which context is to be generated.
• ollamaUrl: The URL for the Ollama service.
• contextualModel: The model used for generating the context.
• options: Additional options for the LLM (Language Model).

Returns:

• A string containing the succinct context for the chunk.
• An error if the context generation fails.

func CreateChunkContextWithPromptTemplate

func CreateChunkContextWithPromptTemplate(promptTpl, wholeDocumentContent string, chunk Chunk, ollamaUrl, contextualModel string, options llm.Options) (string, error)

CreateChunkContextWithPromptTemplate generates a contextual response based on a given prompt template and document content. It interpolates the template with the provided document and chunk content, then uses an LLM to generate a response.

Parameters:

• promptTpl: A string template for the prompt.
• wholeDocumentContent: The content of the entire document.
• chunk: A Chunk struct containing a portion of the document content.
• ollamaUrl: The URL of the LLM service.
• contextualModel: The model to be used for generating the response.
• options: Options for the LLM query.

Returns:

• A string containing the generated response.
• An error if the process fails at any step.

func FindFiles

func FindFiles(dirPath string, ext string) ([]string, error)

FindFiles searches for files with a specific extension in the given root directory and its subdirectories.

Parameters:

• root: The root directory to start the search from.
• ext: The file extension to search for. examples: ".md", ".html", ".txt", "*.*"

Returns: - []string: A slice of file paths that match the given extension. - error: An error if the search encounters any issues.

func ForEachFile

func ForEachFile(dirPath string, ext string, callback func(string) error) ([]string, error)

ForEachFile iterates over all files with a specific extension in a directory and its subdirectories.

Parameters: - dirPath: The root directory to start the search from. - ext: The file extension to search for. - callback: A function to be called for each file found.

Returns: - []string: A slice of file paths that match the given extension. - error: An error if the search encounters any issues.

func GenerateContextFromDocs

func GenerateContextFromDocs(docs []string) string

GenerateContextFromDocs generates the context content from a slice of documents.

Remarks: you can use the generated content to add context to a prompt for an LLM.

Parameters: - docs: a slice of strings representing the documents.

Returns: - string: the generated context content in XML format.

func GetArrayOfContentFiles

func GetArrayOfContentFiles(dirPath string, ext string) ([]string, error)

GetArrayOfContentFiles searches for files with a specific extension in the given directory and its subdirectories.

Parameters: - dirPath: The directory path to start the search from. - ext: The file extension to search for.

Returns: - []string: A slice of file paths that match the given extension. - error: An error if the search encounters any issues.

func GetMapOfContentFiles

func GetMapOfContentFiles(dirPath string, ext string) (map[string]string, error)

GetMapOfContentFiles searches for files with a specific extension in the given directory and its subdirectories.

Parameters: - dirPath: The directory path to start the search from. - ext: The file extension to search for.

Returns: - map[string]string: A map of file paths to their contents, where the keys are the base names of the files and the values are the file contents as strings. - error: An error if the search encounters any issues.

func InterpolateString

func InterpolateString(str string, vars interface{}) (string, error)

func ReadTextFile

func ReadTextFile(path string) (string, error)

ReadTextFile reads the contents of a text file at the given path and returns the contents as a string.

Parameters: - path: the path to the text file.

Returns: - string: the contents of the text file as a string. - error: an error if the file cannot be read.

func SplitAsciiDocBySections

func SplitAsciiDocBySections(content string) []string

func SplitHTMLBySections

func SplitHTMLBySections(content string) ([]string, error)

func SplitMarkdownByLevelSections

func SplitMarkdownByLevelSections(content string, level int) []string

sections := SplitMarkdownByLevelSections(content, 1) // Split by top-level sections

for i, section := range sections {
	fmt.Printf("Section %d:\n%s\n\n", i+1, section)
}

subsections := SplitMarkdownByLevelSections(content, 2) // Split by second-level sections

for i, subsection := range subsections {
	fmt.Printf("Subsection %d:\n%s\n\n", i+1, subsection)
}

func SplitMarkdownBySections

func SplitMarkdownBySections(content string) []string

func SplitTextWithDelimiter

func SplitTextWithDelimiter(text string, delimiter string) []string

SplitTextWithDelimiter splits the given text using the specified delimiter and returns a slice of strings.

Parameters:

• text: The text to be split.
• delimiter: The delimiter used to split the text.

Returns:

• []string: A slice of strings containing the split parts of the text.

func SplitTextWithRegex

func SplitTextWithRegex(text string, regexDelimiter string) []string

SplitTextWithRegex splits the given text using the provided regular expression delimiter. It returns a slice of strings containing the split parts of the text.

func WriteTextFile

func WriteTextFile(path, content string) error

Types

type Chunk

type Chunk struct {
	Header         string
	Content        string
	Level          int
	Prefix         string
	ParentLevel    int
	ParentHeader   string
	ParentPrefix   string
	Lineage        string
	SimpleMetaData string                 // Additional metadata if needed
	Metadata       map[string]interface{} // additional metadata
	KeyWords       []string               // Keywords that could be extracted from the content
}

func ParseMarkdown

func ParseMarkdown(content string) []*Chunk

func ParseMarkdownWithHierarchy

func ParseMarkdownWithHierarchy(content string) []Chunk

func ParseMarkdownWithLineage

func ParseMarkdownWithLineage(content string) []Chunk