scraper

Documentation

Index

• Constants
• func MatchScrapedSceneMovie(qb models.MovieReader, m *models.ScrapedSceneMovie) error
• func MatchScrapedScenePerformer(qb models.PerformerReader, p *models.ScrapedScenePerformer) error
• func MatchScrapedSceneStudio(qb models.StudioReader, s *models.ScrapedSceneStudio) error
• func MatchScrapedSceneTag(qb models.TagReader, s *models.ScrapedSceneTag) error
• type Cache
  • func NewCache(globalConfig GlobalConfig, txnManager models.TransactionManager) (*Cache, error)
  • func (c Cache) ListGalleryScrapers() []*models.Scraper
  • func (c Cache) ListMovieScrapers() []*models.Scraper
  • func (c Cache) ListPerformerScrapers() []*models.Scraper
  • func (c Cache) ListSceneScrapers() []*models.Scraper
  • func (c *Cache) ReloadScrapers() error
  • func (c Cache) ScrapeGallery(scraperID string, gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error)
  • func (c Cache) ScrapeGalleryURL(url string) (*models.ScrapedGallery, error)
  • func (c Cache) ScrapeMovieURL(url string) (*models.ScrapedMovie, error)
  • func (c Cache) ScrapePerformer(scraperID string, scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error)
  • func (c Cache) ScrapePerformerList(scraperID string, query string) ([]*models.ScrapedPerformer, error)
  • func (c Cache) ScrapePerformerURL(url string) (*models.ScrapedPerformer, error)
  • func (c Cache) ScrapeScene(scraperID string, scene models.SceneUpdateInput) (*models.ScrapedScene, error)
  • func (c Cache) ScrapeSceneURL(url string) (*models.ScrapedScene, error)
  • func (c *Cache) UpdateConfig(globalConfig GlobalConfig)
• type GlobalConfig

Constants

const FreeonesScraperID = "builtin_freeones"

FreeonesScraperID is the scraper ID for the built-in Freeones scraper

Functions

func MatchScrapedSceneMovie

func MatchScrapedSceneMovie(qb models.MovieReader, m *models.ScrapedSceneMovie) error

MatchScrapedSceneMovie matches the provided movie with the movies in the database and sets the ID field if one is found.

func MatchScrapedScenePerformer

func MatchScrapedScenePerformer(qb models.PerformerReader, p *models.ScrapedScenePerformer) error

MatchScrapedScenePerformer matches the provided performer with the performers in the database and sets the ID field if one is found.

func MatchScrapedSceneStudio

func MatchScrapedSceneStudio(qb models.StudioReader, s *models.ScrapedSceneStudio) error

MatchScrapedSceneStudio matches the provided studio with the studios in the database and sets the ID field if one is found.

func MatchScrapedSceneTag

func MatchScrapedSceneTag(qb models.TagReader, s *models.ScrapedSceneTag) error

MatchScrapedSceneTag matches the provided tag with the tags in the database and sets the ID field if one is found.

Types

type Cache

type Cache struct {
	// contains filtered or unexported fields
}

Cache stores scraper details.

func NewCache

func NewCache(globalConfig GlobalConfig, txnManager models.TransactionManager) (*Cache, error)

NewCache returns a new Cache loading scraper configurations from the scraper path provided in the global config object. It returns a new instance and an error if the scraper directory could not be loaded.

Scraper configurations are loaded from yml files in the provided scrapers directory and any subdirectories.

func (Cache) ListGalleryScrapers

func (c Cache) ListGalleryScrapers() []*models.Scraper

ListGalleryScrapers returns a list of scrapers that are capable of scraping galleries.

func (Cache) ListMovieScrapers

func (c Cache) ListMovieScrapers() []*models.Scraper

ListMovieScrapers returns a list of scrapers that are capable of scraping scenes.

func (Cache) ListPerformerScrapers

func (c Cache) ListPerformerScrapers() []*models.Scraper

ListPerformerScrapers returns a list of scrapers that are capable of scraping performers.

func (Cache) ListSceneScrapers

func (c Cache) ListSceneScrapers() []*models.Scraper

ListSceneScrapers returns a list of scrapers that are capable of scraping scenes.

func (*Cache) ReloadScrapers

func (c *Cache) ReloadScrapers() error

ReloadScrapers clears the scraper cache and reloads from the scraper path. In the event of an error during loading, the cache will be left empty.

func (Cache) ScrapeGallery

func (c Cache) ScrapeGallery(scraperID string, gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error)

ScrapeGallery uses the scraper with the provided ID to scrape a scene.

func (Cache) ScrapeGalleryURL

func (c Cache) ScrapeGalleryURL(url string) (*models.ScrapedGallery, error)

ScrapeGalleryURL uses the first scraper it finds that matches the URL provided to scrape a scene. If no scrapers are found that matches the URL, then nil is returned.

func (Cache) ScrapeMovieURL

func (c Cache) ScrapeMovieURL(url string) (*models.ScrapedMovie, error)

ScrapeMovieURL uses the first scraper it finds that matches the URL provided to scrape a movie. If no scrapers are found that matches the URL, then nil is returned.

func (Cache) ScrapePerformer

func (c Cache) ScrapePerformer(scraperID string, scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error)

ScrapePerformer uses the scraper with the provided ID to scrape a performer using the provided performer fragment.

func (Cache) ScrapePerformerList

func (c Cache) ScrapePerformerList(scraperID string, query string) ([]*models.ScrapedPerformer, error)

ScrapePerformerList uses the scraper with the provided ID to query for performers using the provided query string. It returns a list of scraped performer data.

func (Cache) ScrapePerformerURL

func (c Cache) ScrapePerformerURL(url string) (*models.ScrapedPerformer, error)

ScrapePerformerURL uses the first scraper it finds that matches the URL provided to scrape a performer. If no scrapers are found that matches the URL, then nil is returned.

func (Cache) ScrapeScene

func (c Cache) ScrapeScene(scraperID string, scene models.SceneUpdateInput) (*models.ScrapedScene, error)

ScrapeScene uses the scraper with the provided ID to scrape a scene.

func (Cache) ScrapeSceneURL

func (c Cache) ScrapeSceneURL(url string) (*models.ScrapedScene, error)

ScrapeSceneURL uses the first scraper it finds that matches the URL provided to scrape a scene. If no scrapers are found that matches the URL, then nil is returned.

func (*Cache) UpdateConfig

func (c *Cache) UpdateConfig(globalConfig GlobalConfig)

UpdateConfig updates the global config for the cache. If the scraper path has changed, ReloadScrapers will need to be called separately.

type GlobalConfig

type GlobalConfig struct {
	// User Agent used when scraping using http.
	UserAgent string

	// Path (file or remote address) to a Chrome CDP instance.
	CDPPath string
	Path    string
}

GlobalConfig contains the global scraper options.