youtube

README

🔦🔴 crawly-live-youtube

Automatically detects when a YouTube channel goes live.

Depends on rubpy/crawly.

Example

An interactive example can be found here: example/example.go.

Documentation

Index

• Variables
• func IsValidChannelID(s string) bool
• func IsValidChannelURL(s string) bool
• func IsValidVideoID(s string) bool
• type ConfigOption
  • func WithClient(client cclient.Client) ConfigOption
  • func WithLogger(logger *slog.Logger) ConfigOption
  • func WithService(service *youtube.Service) ConfigOption
  • func WithSettings(settings CrawlerSettings) ConfigOption
• type Crawler
  • func NewCrawler(opts ...ConfigOption) (*Crawler, error)
  • func (cr *Crawler) ChannelID(channelURL string) (channelID string, ok bool)
  • func (cr *Crawler) CheckLiveVideoState(ctx context.Context, videoID string) (live bool, finished bool, err error)
  • func (cr *Crawler) CheckLiveVideoThumbnail(ctx context.Context, videoID string, thumbnailURL string) (exists bool, err error)
  • func (cr *Crawler) FetchChannelIndex(ctx context.Context, channelURL string) (index *xmlapi.ChannelIndex, err error)
  • func (cr *Crawler) FetchChannelXMLFeed(ctx context.Context, channelID string) (feed *xmlapi.ChannelFeed, err error)
  • func (cr *Crawler) IsTracked(handle Handle) bool
  • func (cr *Crawler) SetSettings(settings CrawlerSettings)
  • func (cr *Crawler) Settings() CrawlerSettings
• type CrawlerSettings
• type EntityData
• type Handle
  • func ChannelID(channelID string) Handle
  • func ChannelURL(channelURL string) Handle
  • func (h Handle) Equal(handle crawly.Handle) bool
  • func (h Handle) String() string
  • func (h Handle) Valid() bool
• type HandleType
  • func (ht HandleType) String() string
• type OrderData
• type VideoCandidate

Variables

var (
	NilConfig  = errors.New("config is nil")
	NilClient  = errors.New("client is nil")
	NilService = errors.New("service is nil")
)

var (
	InvalidChannelID            = errors.New("invalid channel ID")
	InvalidChannelURL           = errors.New("invalid channel URL")
	InvalidVideoID              = errors.New("invalid video ID")
	InvalidVideoThumbnailURL    = errors.New("invalid video thumbnail URL")
	UncertainLiveVideoThumbnail = errors.New("uncertain live video thumbnail status")
)

var DefaultSettings = CrawlerSettings{
	CrawlerSettings: crawly.DefaultCrawlerSettings,

	StopAfterLiveVideos: 1,

	MinimumFetchChannelFeedDelay:       45 * time.Second,
	MaximumCachedNotLivestreamAge:      15 * time.Minute,
	MaximumCachedLivestreamFinishedAge: 60 * time.Minute,
	MinimumCheckVideoDelay:             30 * time.Second,
	MaximumVideoAge:                    60 * 24 * time.Hour,

	CheckVideoTimeout: 10 * time.Second,
}

var (
	ExceededCheckVideoTimeout = errors.New("exceeded check video timeout")
)

Functions

func IsValidChannelID

func IsValidChannelID(s string) bool

func IsValidChannelURL

func IsValidChannelURL(s string) bool

func IsValidVideoID

func IsValidVideoID(s string) bool

Types

type ConfigOption

type ConfigOption func(cfg *config)

func WithClient

func WithClient(client cclient.Client) ConfigOption

func WithLogger

func WithLogger(logger *slog.Logger) ConfigOption

func WithService

func WithService(service *youtube.Service) ConfigOption

func WithSettings

func WithSettings(settings CrawlerSettings) ConfigOption

type Crawler

type Crawler struct {
	crawly.Crawler
	// contains filtered or unexported fields
}

func NewCrawler

func NewCrawler(opts ...ConfigOption) (*Crawler, error)

func (*Crawler) ChannelID

func (cr *Crawler) ChannelID(channelURL string) (channelID string, ok bool)

func (*Crawler) CheckLiveVideoState

func (cr *Crawler) CheckLiveVideoState(ctx context.Context, videoID string) (live bool, finished bool, err error)

func (*Crawler) CheckLiveVideoThumbnail

func (cr *Crawler) CheckLiveVideoThumbnail(ctx context.Context, videoID string, thumbnailURL string) (exists bool, err error)

NOTE: thumbnailURL is an optional 'hint'.

func (*Crawler) FetchChannelIndex

func (cr *Crawler) FetchChannelIndex(ctx context.Context, channelURL string) (index *xmlapi.ChannelIndex, err error)

func (*Crawler) FetchChannelXMLFeed

func (cr *Crawler) FetchChannelXMLFeed(ctx context.Context, channelID string) (feed *xmlapi.ChannelFeed, err error)

func (*Crawler) IsTracked

func (cr *Crawler) IsTracked(handle Handle) bool

func (*Crawler) SetSettings

func (cr *Crawler) SetSettings(settings CrawlerSettings)

func (*Crawler) Settings

func (cr *Crawler) Settings() CrawlerSettings

type CrawlerSettings

type CrawlerSettings struct {
	crawly.CrawlerSettings

	// Stop looking for more live videos after this number is reached (e.g.,
	// if this is set to 1, crawler will not scan remaining videos once it has
	// found one livestream).
	StopAfterLiveVideos int

	MinimumFetchChannelFeedDelay       time.Duration
	MaximumCachedNotLivestreamAge      time.Duration
	MaximumCachedLivestreamFinishedAge time.Duration
	MinimumCheckVideoDelay             time.Duration
	MaximumVideoAge                    time.Duration

	CheckVideoTimeout time.Duration
}

type EntityData

type EntityData struct {
	Live       bool     `json:"live"`
	LiveVideos []string `json:"live_videos"`

	Feed          *xmlapi.ChannelFeed `json:"feed"`
	LastFeedFetch time.Time           `json:"last_feed_fetch"`

	FeedVideoCandidates []VideoCandidate `json:"feed_video_candidates"`
}

type Handle

type Handle struct {
	Type  HandleType
	Value string
}

func ChannelID

func ChannelID(channelID string) Handle

func ChannelURL

func ChannelURL(channelURL string) Handle

func (Handle) Equal

func (h Handle) Equal(handle crawly.Handle) bool

func (Handle) String

func (h Handle) String() string

func (Handle) Valid

func (h Handle) Valid() bool

type HandleType

type HandleType uint

const (
	HandleChannelID HandleType = (iota + 1)
	HandleChannelURL
)

func (HandleType) String

func (ht HandleType) String() string

type OrderData

type OrderData struct{}

type VideoCandidate

type VideoCandidate struct {
	ID          string    `json:"id"`
	ChannelID   string    `json:"channel_id"`
	LastProcess time.Time `json:"last_process"`

	Live             bool      `json:"live"`
	LiveGenuine      bool      `json:"live_genuine"`
	LiveCheckAttempt int       `json:"live_check_attempt"`
	LastLive         time.Time `json:"last_live"`

	LivestreamFinished     bool      `json:"livestream_finished"`
	LastLivestreamFinished time.Time `json:"last_livestream_finished"`

	NotLivestream     bool      `json:"not_livestream"`
	LastNotLivestream time.Time `json:"last_not_livestream"`
}