scraper

Documentation

Index

• func GetConf() *config
• type YoutubeApiService
• type YoutubeSeed
  • func GetYoutubeSeed(SearchResource []string, SearchKey string, OrderBy string, ...) *YoutubeSeed
  • func (s *YoutubeSeed) InitialiseScraper(ctx context.Context) error
  • func (s *YoutubeSeed) MakeApiCall(ctx context.Context, youtubeSearchService *youtube.SearchService) (*youtube.SearchListResponse, error)
  • func (s *YoutubeSeed) PersistVideoData(ctx context.Context, searchRes *youtube.SearchListResponse) error

Functions

func GetConf

func GetConf() *config

Types

type YoutubeApiService

type YoutubeApiService interface {
	// InitialiseScraper orchestrates periodic call to scrape YouTube data after a fixed interval
	InitialiseScraper(ctx context.Context) error
	// MakeApiCall executes the api call to youtubeV3 api
	MakeApiCall(ctx context.Context, service *youtube.SearchService) (*youtube.SearchListResponse, error)
	// PersistVideoData to save the video meta data in postgres
	PersistVideoData(ctx context.Context, searchRes *youtube.SearchListResponse) error
}

type YoutubeSeed

type YoutubeSeed struct {
	// contains filtered or unexported fields
}

func GetYoutubeSeed

func GetYoutubeSeed(SearchResource []string, SearchKey string, OrderBy string, MaxResultPerCall int, ApiKey []string, LastSearchTime time.Time,
	dbClient *db.Handler) *YoutubeSeed

func (*YoutubeSeed) InitialiseScraper

func (s *YoutubeSeed) InitialiseScraper(ctx context.Context) error

func (*YoutubeSeed) MakeApiCall

func (s *YoutubeSeed) MakeApiCall(ctx context.Context,
	youtubeSearchService *youtube.SearchService) (*youtube.SearchListResponse, error)

func (*YoutubeSeed) PersistVideoData

func (s *YoutubeSeed) PersistVideoData(ctx context.Context, searchRes *youtube.SearchListResponse) error