katsuragi

README

A Go toolkit for web content processing, analysis, and SEO optimization, offering utilities to efficiently extract titles, descriptions and favicons.

[!NOTE] Each method is thoroughly tested and optimized for performance, but the package is still in development and may contain unseen bugs. Please don't hesitate to report any issues you encounter!

Table of Contents

Features

• LRU Caching
• Timeout
• User-Agent

Installation

go get github.com/devnyxie/katsuragi

Usage

Title

import (
	. "github.com/devnyxie/katsuragi"
)

func main() {
  // Create a new fetcher with a timeout of 3 seconds and a cache capacity of 10
  fetcher := NewFetcher(
    &FetcherProps{
      Timeout:       3000, // 3 seconds
      CacheCap: 10, // 10 Network Requests will be cached
    },
  )

  defer fetcher.ClearCache()

  // Get website's title
  title, err := fetcher.GeTitle("https://www.example.com")
}

Description

...
  // Get website's description
  description, err := fetcher.GetDescription("https://www.example.com")
...

Favicons

...
  // Get website's favicons
  favicons, err := fetcher.GetFavicons("https://www.example.com")
...

Local Development

Testing

go test -v

Documentation

Index

• type Fetcher
  • func NewFetcher(props *FetcherProps) *Fetcher
  • func (f *Fetcher) ClearCache()
  • func (f *Fetcher) GetDescription(url string) (string, error)
  • func (f *Fetcher) GetFavicons(url string) ([]string, error)
  • func (f *Fetcher) GetFromCache(url string) (*html.Node, bool, error)
  • func (f *Fetcher) GetTitle(url string) (string, error)
• type FetcherProps

Types

type Fetcher

type Fetcher struct {
	// contains filtered or unexported fields
}

func NewFetcher

func NewFetcher(props *FetcherProps) *Fetcher

func (*Fetcher) ClearCache

func (f *Fetcher) ClearCache()

func (*Fetcher) GetDescription

func (f *Fetcher) GetDescription(url string) (string, error)

func (*Fetcher) GetFavicons

func (f *Fetcher) GetFavicons(url string) ([]string, error)

GetFavicon fetches the favicon of a webpage given its URL

func (*Fetcher) GetFromCache

func (f *Fetcher) GetFromCache(url string) (*html.Node, bool, error)

func (*Fetcher) GetTitle

func (f *Fetcher) GetTitle(url string) (string, error)

GetTitle fetches the title of a webpage given its URL

type FetcherProps

type FetcherProps struct {
	UserAgent string
	Timeout   time.Duration //ms
	CacheCap  int
}