content

Documentation

Index

• func ChunkText(text string, chunkSize, overlap int) []string
• func FindFiles(dirPath string, ext string) ([]string, error)
• func ForEachFile(dirPath string, ext string, callback func(string) error) ([]string, error)
• func GenerateContextFromDocs(docs []string) string
• func GetArrayOfContentFiles(dirPath string, ext string) ([]string, error)
• func GetMapOfContentFiles(dirPath string, ext string) (map[string]string, error)
• func InterpolateString(str string, vars interface{}) (string, error)
• func ReadTextFile(path string) (string, error)
• func SplitAsciiDocBySections(content string) []string
• func SplitHTMLBySections(content string) ([]string, error)
• func SplitMarkdownByLevelSections(content string, level int) []string
• func SplitMarkdownBySections(content string) []string
• func SplitTextWithDelimiter(text string, delimiter string) []string
• func SplitTextWithRegex(text string, regexDelimiter string) []string
• type Chunk
  • func ParseMarkdown(content string) []*Chunk
  • func ParseMarkdownWithHierarchy(content string) []Chunk
  • func ParseMarkdownWithLineage(content string) []Chunk

Functions

func ChunkText

func ChunkText(text string, chunkSize, overlap int) []string

ChunkText takes a text string and divides it into chunks of a specified size with a given overlap. It returns a slice of strings, where each string represents a chunk of the original text.

Parameters:

• text: The input text to be chunked.
• chunkSize: The size of each chunk.
• overlap: The amount of overlap between consecutive chunks.

Returns:

• []string: A slice of strings representing the chunks of the original text.

func FindFiles

func FindFiles(dirPath string, ext string) ([]string, error)

FindFiles searches for files with a specific extension in the given root directory and its subdirectories.

Parameters:

• root: The root directory to start the search from.
• ext: The file extension to search for. examples: ".md", ".html", ".txt", "*.*"

Returns: - []string: A slice of file paths that match the given extension. - error: An error if the search encounters any issues.

func ForEachFile

func ForEachFile(dirPath string, ext string, callback func(string) error) ([]string, error)

ForEachFile iterates over all files with a specific extension in a directory and its subdirectories.

Parameters: - dirPath: The root directory to start the search from. - ext: The file extension to search for. - callback: A function to be called for each file found.

Returns: - []string: A slice of file paths that match the given extension. - error: An error if the search encounters any issues.

func GenerateContextFromDocs

func GenerateContextFromDocs(docs []string) string

GenerateContextFromDocs generates the context content from a slice of documents.

Remarks: you can use the generated content to add context to a prompt for an LLM.

Parameters: - docs: a slice of strings representing the documents.

Returns: - string: the generated context content in XML format.

func GetArrayOfContentFiles

func GetArrayOfContentFiles(dirPath string, ext string) ([]string, error)

GetArrayOfContentFiles searches for files with a specific extension in the given directory and its subdirectories.

Parameters: - dirPath: The directory path to start the search from. - ext: The file extension to search for.

Returns: - []string: A slice of file paths that match the given extension. - error: An error if the search encounters any issues.

func GetMapOfContentFiles

func GetMapOfContentFiles(dirPath string, ext string) (map[string]string, error)

GetMapOfContentFiles searches for files with a specific extension in the given directory and its subdirectories.

Parameters: - dirPath: The directory path to start the search from. - ext: The file extension to search for.

Returns: - map[string]string: A map of file paths to their contents, where the keys are the base names of the files and the values are the file contents as strings. - error: An error if the search encounters any issues.

func InterpolateString

func InterpolateString(str string, vars interface{}) (string, error)

func ReadTextFile

func ReadTextFile(path string) (string, error)

ReadTextFile reads the contents of a text file at the given path and returns the contents as a string.

Parameters: - path: the path to the text file.

Returns: - string: the contents of the text file as a string. - error: an error if the file cannot be read.

func SplitAsciiDocBySections

func SplitAsciiDocBySections(content string) []string

func SplitHTMLBySections

func SplitHTMLBySections(content string) ([]string, error)

func SplitMarkdownByLevelSections

func SplitMarkdownByLevelSections(content string, level int) []string

sections := SplitMarkdownByLevelSections(content, 1) // Split by top-level sections

for i, section := range sections {
	fmt.Printf("Section %d:\n%s\n\n", i+1, section)
}

subsections := SplitMarkdownByLevelSections(content, 2) // Split by second-level sections

for i, subsection := range subsections {
	fmt.Printf("Subsection %d:\n%s\n\n", i+1, subsection)
}

func SplitMarkdownBySections

func SplitMarkdownBySections(content string) []string

func SplitTextWithDelimiter

func SplitTextWithDelimiter(text string, delimiter string) []string

SplitTextWithDelimiter splits the given text using the specified delimiter and returns a slice of strings.

Parameters:

• text: The text to be split.
• delimiter: The delimiter used to split the text.

Returns:

• []string: A slice of strings containing the split parts of the text.

func SplitTextWithRegex

func SplitTextWithRegex(text string, regexDelimiter string) []string

SplitTextWithRegex splits the given text using the provided regular expression delimiter. It returns a slice of strings containing the split parts of the text.

Types

type Chunk

type Chunk struct {
	Header       string
	Content      string
	Level        int
	Prefix       string
	ParentLevel  int
	ParentHeader string
	ParentPrefix string
	Lineage      string
	MetaData     string   // Additional metadata if needed
	KeyWords     []string // Keywords that could be extracted from the content
}

func ParseMarkdown

func ParseMarkdown(content string) []*Chunk

func ParseMarkdownWithHierarchy

func ParseMarkdownWithHierarchy(content string) []Chunk

func ParseMarkdownWithLineage

func ParseMarkdownWithLineage(content string) []Chunk