magictext

README

magictext

Generate a high-quality summary of a given text input.

TODO

• Support goroutines to improve performance
• Generate separate levels of summary and write to different files
• Support explicitly set OpenAI APIKey
• Generate parent summary with fixed number of chunks
• Calculate tokens with tiktoken-go
• Add retry logic for goroutines
• fix token regex performance issues
• remove video header and tail from rst file
• return the error instance, don't call log's fatal function directly
• Write project documentation

Documentation

Index

• Constants
• Variables
• func CountTokens(text string) int
• func DumpChunksToJSON(filename string, chunks interface{}) error
• func DumpSummary(filename, summary string, captionSummaries []*CaptionSummary) error
• func ExtractNouns(text string) (string, error)
• func GenerateTitle(text string) (string, error)
• func SplitText(text string, chunkSize, chunkOverlap int) ([]string, error)
• type CaptionChunk
  • func NewCaptionChunk(seq int, text string, from, to time.Time) *CaptionChunk
  • func SplitSubtitle(subtitle subtitles.Subtitle) ([]*CaptionChunk, error)
  • func (c CaptionChunk) String() string
• type CaptionSummary
  • func GenerateSummaryBySubtitle(topic string, subtitle subtitles.Subtitle) ([]*CaptionSummary, string, error)
  • func (cs CaptionSummary) FromInSeconds() int
  • func (cs CaptionSummary) FromInString() string
  • func (cs CaptionSummary) ToInSeconds() int
  • func (cs CaptionSummary) ToInString() string
• type Chunk
  • func NewChunk(seq int, text string) *Chunk
  • func (c *Chunk) String() string
• type ChunkSlice
  • func (cs ChunkSlice) String() string
  • func (cs ChunkSlice) Text() string
  • func (cs ChunkSlice) Tokens() int
• type Summary
• type TextChunk

Constants

const (
	MaxReqTokens512  = 512
	MaxReqTokens2048 = 2048
)

const (
	MaxRetryTimes = 3
	SleepSeconds  = 3
)

const (
	DefaultPrompt = "`reset` `no quotes` `no explanations` `no prompt` `no self-reference` `no apologies` `no filler` `just answer` "

	GenerateSummaryPrompt = DefaultPrompt + `
I will give you text content, you will rewrite it and output that in a short summarized version of my text. Keep the meaning the same. Ensure that the revised content has fewer characters than the original text and no more than 150 Chinese words.

Only give me the output and nothing else. Now, using the concepts above, summarize the following text. Respond in Chinese language.

[text]

%s

[output]
`

	GenerateSummaryPromptWithTopic = DefaultPrompt + `
I will give you text content, you will rewrite it and output that in a short summarized version of my text. Keep the meaning the same. Ensure that the revised content has fewer characters than the original text and no more than 150 Chinese words.

When generating text summaries, expand around the following topics as much as possible: %s` + `

Only give me the output and nothing else. Now, using the concepts above, summarize the following text. Respond in Chinese language.

[text]

%s

[output]
`

	GenerateTitlePrompt = DefaultPrompt + `
Create a title for the paragraph below. The title should be concise and to the point. The number of characters should not exceed 15 Chinese characters. This title will be used as the title of the video. Respond in Chinese language.

[text]

%s
	
[output]
`

	ExtractNounsPrompt = DefaultPrompt + `
Find all user names, company names, product names, course names, and book names from the following text, and output them in the json format. Respond in Chinese language.

[output format]
{
	"usernames": [],
	"company_names": [],
	"product_names": [],
	"course_names": [],
	"book_names": [],
}

[text]

%s

[output]
`
)

const (
	MaxChunksPerGroup = 3 // TODO: this variable should be calculated dynamically
	MaxConcurrent     = 5
)

Variables

var (
	Debug        = false
	MockOpenAI   = false
	OpenAIClient *openai.Client
	TikToken     *tiktoken.Tiktoken
)

Functions

func CountTokens

func CountTokens(text string) int

func DumpChunksToJSON

func DumpChunksToJSON(filename string, chunks interface{}) error

DumpChunksToJSON writes chunk slice to the given file.

func DumpSummary

func DumpSummary(filename, summary string, captionSummaries []*CaptionSummary) error

func ExtractNouns

func ExtractNouns(text string) (string, error)

ExtractNouns extracts nouns from a string, the max length of input text is 2048, the output is a json string, see following example for more information.

Output string:

{
   "usernames": ["吴三桂", "皇太极", "弘历"],
   "company_names": ["得到"],
   "product_names": [],
   "course_names": ["硅谷来信"],
   "book_names": ["万历十五年", "湘行散记", "货币未来"]
}

func GenerateTitle

func GenerateTitle(text string) (string, error)

GenerateTitle generates a title for the given text, the max length of input text is 512.

func SplitText

func SplitText(text string, chunkSize, chunkOverlap int) ([]string, error)

Types

type CaptionChunk

type CaptionChunk struct {
	From time.Time `json:"from"`
	To   time.Time `json:"to"`
	TextChunk
}

func NewCaptionChunk

func NewCaptionChunk(seq int, text string, from, to time.Time) *CaptionChunk

func SplitSubtitle

func SplitSubtitle(subtitle subtitles.Subtitle) ([]*CaptionChunk, error)

func (CaptionChunk) String

func (c CaptionChunk) String() string

type CaptionSummary

type CaptionSummary struct {
	From time.Time
	To   time.Time
	Summary
}

func GenerateSummaryBySubtitle

func GenerateSummaryBySubtitle(topic string, subtitle subtitles.Subtitle) ([]*CaptionSummary, string, error)

GenerateSummaryBySubtitle generates a summary for the given subtitles

func (CaptionSummary) FromInSeconds

func (cs CaptionSummary) FromInSeconds() int

func (CaptionSummary) FromInString

func (cs CaptionSummary) FromInString() string

func (CaptionSummary) ToInSeconds

func (cs CaptionSummary) ToInSeconds() int

func (CaptionSummary) ToInString

func (cs CaptionSummary) ToInString() string

type Chunk

type Chunk struct {
	ID       string     `json:"id"`
	Height   int        `json:"height"`
	Seq      int        `json:"seq"`
	Text     string     `json:"text"`
	Tokens   int        `json:"tokens"`
	Children ChunkSlice `json:"children"`
}

func NewChunk

func NewChunk(seq int, text string) *Chunk

func (*Chunk) String

func (c *Chunk) String() string

type ChunkSlice

type ChunkSlice []*Chunk

func (ChunkSlice) String

func (cs ChunkSlice) String() string

func (ChunkSlice) Text

func (cs ChunkSlice) Text() string

func (ChunkSlice) Tokens

func (cs ChunkSlice) Tokens() int

type Summary

type Summary struct {
	ID   string
	Seq  int
	Text string
}

type TextChunk

type TextChunk struct {
	ID     string `json:"id"`
	Seq    int    `json:"seq"`
	Text   string `json:"text"`
	Tokens int    `json:"tokens"`
}